{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2cd2accf-5877-4136-a243-7a33a13ce2b4",
   "metadata": {},
   "source": [
    "<img src=\"http://developer.download.nvidia.com/notebooks/dlsw-notebooks/tensorrt_torchtrt_efficientnet/nvidia_logo.png\" width=\"90px\">\n",
    "\n",
    "# Pyspark TensorFlow Inference\n",
    "\n",
    "### Text Classification\n",
    "In this notebook, we demonstrate training a model to perform sentiment analysis, and using the trained model for distributed inference.  \n",
    "Based on: https://www.tensorflow.org/tutorials/keras/text_classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc72d0ed",
   "metadata": {},
   "source": [
    "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76f0f5df-502f-444e-b2ee-1122e1dea870",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-02-04 14:05:12.899608: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2025-02-04 14:05:12.907256: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2025-02-04 14:05:12.915374: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2025-02-04 14:05:12.917743: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2025-02-04 14:05:12.924372: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2025-02-04 14:05:13.295411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import shutil\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a364ad5f-b269-45b5-ab8b-d8f34fb642b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.17.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1738706713.692042 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706713.716276 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706713.719037 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "\n",
    "# Enable GPU memory growth\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    except RuntimeError as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b64bb471",
   "metadata": {},
   "source": [
    "### Download and explore the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d229c1b6-3967-46b5-9ea8-68f4b42dd211",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset(\"imdb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "88f9a92e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create directories for our data\n",
    "base_dir = \"spark-dl-datasets/imdb\"\n",
    "if os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False):\n",
    "    # For databricks, use the driver disk rather than Workspace (much faster)\n",
    "    base_dir = \"/local_disk0/\" + base_dir\n",
    "\n",
    "train_dir = base_dir + \"/train\"\n",
    "test_dir = base_dir + \"/test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3f984d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create directories for positive (1) and negative (0) reviews\n",
    "for split in [\"train\", \"test\"]:\n",
    "    split_dir = os.path.join(base_dir, split)\n",
    "    pos_dir = split_dir + \"/pos\"\n",
    "    neg_dir = split_dir + \"/neg\"\n",
    "\n",
    "    os.makedirs(pos_dir, exist_ok=True)\n",
    "    os.makedirs(neg_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6cd2328a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_reviews_to_files(dataset_split, split_name):\n",
    "    for idx, example in enumerate(dataset_split):\n",
    "        label_dir = \"pos\" if example[\"label\"] == 1 else \"neg\"\n",
    "        dir_path = os.path.join(base_dir, split_name, label_dir)\n",
    "\n",
    "        file_path = dir_path + f\"/review_{idx}.txt\"\n",
    "        with open(file_path, \"w\", encoding=\"utf-8\") as f:\n",
    "            f.write(example[\"text\"])\n",
    "\n",
    "# Write train and test sets\n",
    "write_reviews_to_files(dataset[\"train\"], \"train\")\n",
    "write_reviews_to_files(dataset[\"test\"], \"test\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b02fde64",
   "metadata": {},
   "source": [
    "There are 25,000 examples in the training folder, of which we will use 80% (or 20,000) for training, and 5,000 for validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5c357f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 25000 files belonging to 2 classes.\n",
      "Using 20000 files for training.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0000 00:00:1738706719.326625 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706719.329542 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706719.332409 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706719.451656 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706719.452700 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1738706719.453630 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2025-02-04 14:05:19.454569: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40337 MB memory:  -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 25000 files belonging to 2 classes.\n",
      "Using 5000 files for validation.\n",
      "Found 25000 files belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "seed = 42\n",
    "\n",
    "raw_train_ds = tf.keras.utils.text_dataset_from_directory(\n",
    "    str(train_dir),\n",
    "    batch_size=batch_size,\n",
    "    validation_split=0.2,\n",
    "    subset=\"training\",\n",
    "    seed=seed,\n",
    ")\n",
    "\n",
    "raw_val_ds = tf.keras.utils.text_dataset_from_directory(\n",
    "    str(train_dir),\n",
    "    batch_size=batch_size,\n",
    "    validation_split=0.2,\n",
    "    subset=\"validation\",\n",
    "    seed=seed,\n",
    ")\n",
    "\n",
    "raw_test_ds = tf.keras.utils.text_dataset_from_directory(\n",
    "    str(test_dir),\n",
    "    batch_size=batch_size\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02994994",
   "metadata": {},
   "source": [
    "We can take a look at a sample of the dataset (note that OUT_OF_RANGE errors are safe to ignore: https://github.com/tensorflow/tensorflow/issues/62963):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1d528a95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review b'I was really, really disappointed with this movie. it started really well, and built up some great atmosphere and suspense, but when it finally got round to revealing the \"monster\"...it turned out to be just some psycho with skin problems......again. Whoop-de-do. Yet another nutjob movie...like we don\\'t already have enough of them.<br /><br />To be fair, the \"creep\" is genuinely unsettling to look at, and the way he moves and the strange sounds he makes are pretty creepy, but I\\'m sick of renting film like this only to discover that the monster is human, albeit a twisted, demented, freakish one. When I saw all the tell-tale rats early on I was hoping for some kind of freaky rat-monster hybrid thing...it was such a let down when the Creep was revealed.<br /><br />On top of this, some of the stuff in this movie makes no sense. (Spoiler) <br /><br />Why the hell does the Creep kill the security Guard? Whats the point, apart from sticking a great honking sign up that says \"HI I\\'m A PSYCHO AND I LIVE DOWN HERE!\"? Its stupid, and only seems to happen to prevent Franka Potente\\'s character from getting help.<br /><br />what the hells he been eating down there? I got the impression he was effectively walled in, and only the unexpected opening into that tunnel section let him loose...so has he been munching rats all that time, and if so why do they hang around him so much? Why is he so damn hard to kill? He\\'s thin, malnourished and not exactly at peak performance...but seems to keep going despite injuries that are equivalent to those that .cripple the non-psycho characters in the film.<br /><br />The DVD commentary says we are intended to empathise with Creep, but I just find him loathsome. Its an effective enough movie, but it wasted so many opportunities that it makes me sick.'\n",
      "Label 0\n",
      "Review b\"This has the absolute worst performance from Robert Duval who sounds just like William Buckley throughout the entire film. His hammy melodramatic acting takes away from any dramatic interest. I'm not sure if this was deliberate scene stealing or inadvertent but it's the only thing I can recall from a truly forgettable film. This picture should be shown in every amateur acting class of an example of what not to do. Thank God, Duvall went on to bigger and better things and stopped trying to effect a cultured accent. He is a good character actor but that's about it. Klaus is so much better. His performance is muted and noteworthy.\"\n",
      "Label 0\n",
      "Review b'A long time ago, in a galaxy far, far away.....There was a boy who was only two years old when the original \"Star Wars\" film was released. He doesn\\'t remember first seeing the movie, but he also doesn\\'t remember life before it. He does remember the first \"Star Wars\" themed gift he got...a shoebox full of action figures from the original set. He was too young to fully appreciate how special that gift would be. But years later, he would get what to this day goes down as one of the best gifts he\\'s ever received: another box full of action figures, ten of the final twelve he needed to complete his collection. It\\'s now legendary in this boy\\'s family how the last action figure he needed, Anakin Skywalker, stopped being produced and carried in stores, and how this boy went for about ten years (until he got into college) trying to track one down and finally bought it from someone on his dorm floor for a bag of beer nuggets (don\\'t ask...it\\'s a Northern Illinois University thing).<br /><br />I can\\'t review \"Star Wars\" as a movie. It represents absolutely everything good, fun and magical about my childhood. There\\'s no separating it in my mind from Christmases, birthdays, summers and winters growing up. In the winter, my friends and I would build snow forts and pretend we were on Hoth (I was always Han Solo). My friends\\' dad built them a kick-ass tree house, and that served as the Ewok village. They also had a huge pine tree whose bottom branches were high enough to create a sort of cave underneath it, and this made a great spot to pretend we were in Yoda\\'s home. I am unabashedly dorky when it comes to \"Star Wars\" and I think people either just understand that or they don\\'t. I don\\'t get the appeal of \"Lord of the Rings\" or \"Star Trek\" but I understand the rabid flocks of fans that follow them because I am a rabid fan of George Lucas\\'s films.<br /><br />I feel no need to defend my opinion of these movies as some of the greatest of all time. Every time I put them in the DVD player, I feel like I\\'m eight years old again, when life was simple and the biggest problem I had was figuring out how I was going to track down a figure of Anakin Skywalker.<br /><br />Grade (for the entire trilogy): A+'\n",
      "Label 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-02-04 14:05:20.533703: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n"
     ]
    }
   ],
   "source": [
    "for text_batch, label_batch in raw_train_ds.take(1):\n",
    "    for i in range(3):\n",
    "        print(\"Review\", text_batch.numpy()[i])\n",
    "        print(\"Label\", label_batch.numpy()[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bca98b1",
   "metadata": {},
   "source": [
    "Notice the reviews contain raw text (with punctuation and occasional HTML tags like \\<br/>\\). We will show how to handle these in the following section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f8921ed2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label 0 corresponds to neg\n",
      "Label 1 corresponds to pos\n"
     ]
    }
   ],
   "source": [
    "print(\"Label 0 corresponds to\", raw_train_ds.class_names[0])\n",
    "print(\"Label 1 corresponds to\", raw_train_ds.class_names[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6cf0e47",
   "metadata": {},
   "source": [
    "### Prepare the dataset for training\n",
    "\n",
    "Next, we will standardize, tokenize, and vectorize the data using the tf.keras.layers.TextVectorization layer.  \n",
    "We will write a custom standardization function to remove the HTML."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cb141709-fcc1-4cee-bc98-9c89aaba8648",
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_standardization(input_data):\n",
    "    lowercase = tf.strings.lower(input_data)\n",
    "    stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n",
    "    return tf.strings.regex_replace(\n",
    "        stripped_html, \"[%s]\" % re.escape(string.punctuation), \"\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35e36a2",
   "metadata": {},
   "source": [
    "Next, we will create a TextVectorization layer to standardize, tokenize, and vectorize our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d4e80ea9-536a-4ebc-8b35-1eca73dbba7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 10000\n",
    "sequence_length = 250\n",
    "\n",
    "vectorize_layer = layers.TextVectorization(\n",
    "    standardize=custom_standardization,\n",
    "    max_tokens=max_features,\n",
    "    output_mode=\"int\",\n",
    "    output_sequence_length=sequence_length,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "879fbc3f",
   "metadata": {},
   "source": [
    "Next, we will call adapt to fit the state of the preprocessing layer to the dataset. This will cause the model to build an index of strings to integers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ad1e5d81-7dae-4b08-b520-ca45501b9510",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-02-04 14:05:22.003236: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n"
     ]
    }
   ],
   "source": [
    "# Make a text-only dataset (without labels), then call adapt\n",
    "train_text = raw_train_ds.map(lambda x, y: x)\n",
    "vectorize_layer.adapt(train_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad1e5d81-7dae-4b08-b520-ca45501b9510",
   "metadata": {},
   "source": [
    "Let's create a function to see the result of using this layer to preprocess some data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "80f243f5-edd3-4e1c-bddc-abc1cc6673ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_text(text, label):\n",
    "    text = tf.expand_dims(text, -1)\n",
    "    return vectorize_layer(text), label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8f37e95c-515c-4edb-a1ee-fc47be5df4b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review tf.Tensor(b\"To describe this film as garbage is unfair. At least rooting through garbage can be an absorbing hobby. This flick was neither absorbing nor entertaining.<br /><br />Kevin Bacon can act superbly given the chance, so no doubt had an IRS bill to settle when he agreed to this dire screenplay. The mad scientist story of 'Hollow Man' has been told before, been told better, and been told without resorting to so many ludicrously expensive special effects.<br /><br />Most of those special effects seem to be built around the transparent anatomical dolls of men, women and dogs you could buy in the early seventies. In the UK they were marketed as 'The Transparent Man (/Woman/Dog)' which is maybe where they got the title for this film.<br /><br />Clever special effects, dire script, non-existent plot.<br /><br />\", shape=(), dtype=string)\n",
      "Label neg\n",
      "Vectorized review (<tf.Tensor: shape=(1, 250), dtype=int64, numpy=\n",
      "array([[   6, 1507,   11,   19,   14, 1184,    7, 5230,   30,  217, 5821,\n",
      "         139, 1184,   68,   26,   33, 6676,    1,   11,  512,   13, 1078,\n",
      "        6676,  888,  439, 1727, 5292,   68,  503, 3597,  333,    2,  558,\n",
      "          37,   56,  797,   64,   33, 8270,  978,    6, 3956,   51,   27,\n",
      "        4531,    6,   11, 3756,  907,    2, 1106, 1660,   63,    5, 3514,\n",
      "         134,   43,   74,  566,  155,   74,  566,  122,    3,   74,  566,\n",
      "         204,    1,    6,   37,  106,    1, 3152,  307,  293,   88,    5,\n",
      "         143,  307,  293,  294,    6,   26, 2250,  183,    2, 7541,    1,\n",
      "        4379,    5,  352,  362,    3, 2312,   22,   99,  756,    8,    2,\n",
      "         402, 3887,    8,    2, 2142,   34,   65,    1,   14,    2, 7541,\n",
      "         134,    1,   61,    7,  271,  111,   34,  182,    2,  409,   15,\n",
      "          11,   19, 1066,  307,  293, 3756,  223, 2939,  112,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0]])>, <tf.Tensor: shape=(), dtype=int32, numpy=0>)\n"
     ]
    }
   ],
   "source": [
    "# retrieve a batch (of 32 reviews and labels) from the dataset\n",
    "text_batch, label_batch = next(iter(raw_train_ds))\n",
    "first_review, first_label = text_batch[0], label_batch[0]\n",
    "print(\"Review\", first_review)\n",
    "print(\"Label\", raw_train_ds.class_names[first_label])\n",
    "print(\"Vectorized review\", vectorize_text(first_review, first_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "680f53bb",
   "metadata": {},
   "source": [
    "We can lookup the token (string) that each integer corresponds to by calling .get_vocabulary() on the layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "60c9208a-39ac-4e6c-a603-61038cdf3d10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1287 --->  nowhere\n",
      " 313 --->  house\n",
      "Vocabulary size: 10000\n"
     ]
    }
   ],
   "source": [
    "print(\"1287 ---> \",vectorize_layer.get_vocabulary()[1287])\n",
    "print(\" 313 ---> \",vectorize_layer.get_vocabulary()[313])\n",
    "print('Vocabulary size: {}'.format(len(vectorize_layer.get_vocabulary())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3cf90d4b-8dae-44b2-b32b-80cb0092c430",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = raw_train_ds.map(vectorize_text)\n",
    "val_ds = raw_val_ds.map(vectorize_text)\n",
    "test_ds = raw_test_ds.map(vectorize_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3db3f77",
   "metadata": {},
   "source": [
    "### Configure the dataset for performance\n",
    "\n",
    "These are two important methods you should use when loading data to make sure that I/O does not become blocking.\n",
    "\n",
    "`.cache()` keeps data in memory after it's loaded off disk. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache, which is more efficient to read than many small files.\n",
    "\n",
    "`.prefetch()` overlaps data preprocessing and model execution while training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "115a5aba-8a00-458f-be25-0aae9f55de22",
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTOTUNE = tf.data.AUTOTUNE\n",
    "\n",
    "train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
    "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
    "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d6d6692",
   "metadata": {},
   "source": [
    "### Create the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d64f4495-102d-4244-9b42-1ba9976a366e",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3dc95d22-935f-4091-b0ee-da95174eb9a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)           │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)               │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ global_average_pooling1d        │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePooling1D</span>)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ ?                      │   <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ embedding (\u001b[38;5;33mEmbedding\u001b[0m)           │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (\u001b[38;5;33mDropout\u001b[0m)               │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ global_average_pooling1d        │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "│ (\u001b[38;5;33mGlobalAveragePooling1D\u001b[0m)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout_1 (\u001b[38;5;33mDropout\u001b[0m)             │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ ?                      │   \u001b[38;5;34m0\u001b[0m (unbuilt) │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = tf.keras.Sequential([\n",
    "  layers.Embedding(max_features, embedding_dim),\n",
    "  layers.Dropout(0.2),\n",
    "  layers.GlobalAveragePooling1D(),\n",
    "  layers.Dropout(0.2),\n",
    "  layers.Dense(1, activation='sigmoid')])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d9059b93-7666-46db-bf15-517c4c205df9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=losses.BinaryCrossentropy(),\n",
    "              optimizer='adam',\n",
    "              metrics=[tf.metrics.BinaryAccuracy(threshold=0.5)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8b66d33",
   "metadata": {},
   "source": [
    "#### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b1d5959f-1bd8-48da-9815-8239599519b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1738706722.621647 3744883 service.cc:146] XLA service 0x334cd320 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1738706722.621667 3744883 service.cc:154]   StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n",
      "2025-02-04 14:05:22.635317: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2025-02-04 14:05:22.689182: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m262/625\u001b[0m \u001b[32m━━━━━━━━\u001b[0m\u001b[37m━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 578us/step - binary_accuracy: 0.5299 - loss: 0.6904"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0000 00:00:1738706723.175401 3744883 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - binary_accuracy: 0.5692 - loss: 0.6832 - val_binary_accuracy: 0.7020 - val_loss: 0.6195\n",
      "Epoch 2/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 455us/step - binary_accuracy: 0.7588 - loss: 0.5825 - val_binary_accuracy: 0.7954 - val_loss: 0.5009\n",
      "Epoch 3/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 536us/step - binary_accuracy: 0.8293 - loss: 0.4681 - val_binary_accuracy: 0.8352 - val_loss: 0.4253\n",
      "Epoch 4/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 516us/step - binary_accuracy: 0.8523 - loss: 0.3967 - val_binary_accuracy: 0.8516 - val_loss: 0.3802\n",
      "Epoch 5/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 448us/step - binary_accuracy: 0.8692 - loss: 0.3524 - val_binary_accuracy: 0.8592 - val_loss: 0.3522\n",
      "Epoch 6/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 530us/step - binary_accuracy: 0.8810 - loss: 0.3199 - val_binary_accuracy: 0.8658 - val_loss: 0.3324\n",
      "Epoch 7/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 489us/step - binary_accuracy: 0.8919 - loss: 0.2945 - val_binary_accuracy: 0.8666 - val_loss: 0.3188\n",
      "Epoch 8/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 509us/step - binary_accuracy: 0.8975 - loss: 0.2744 - val_binary_accuracy: 0.8720 - val_loss: 0.3085\n",
      "Epoch 9/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 389us/step - binary_accuracy: 0.9042 - loss: 0.2565 - val_binary_accuracy: 0.8756 - val_loss: 0.3017\n",
      "Epoch 10/10\n",
      "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 410us/step - binary_accuracy: 0.9121 - loss: 0.2409 - val_binary_accuracy: 0.8750 - val_loss: 0.2972\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "history = model.fit(\n",
    "    train_ds,\n",
    "    validation_data=val_ds,\n",
    "    epochs=epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c8d8f2a",
   "metadata": {},
   "source": [
    "#### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "656afe07-354f-4ff2-8e3e-d02bad6c5958",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 573us/step - binary_accuracy: 0.8719 - loss: 0.3147\n",
      "Loss:  0.3172186613082886\n",
      "Accuracy:  0.8701599836349487\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy = model.evaluate(test_ds)\n",
    "\n",
    "print(\"Loss: \", loss)\n",
    "print(\"Accuracy: \", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2a307ce",
   "metadata": {},
   "source": [
    "Create a plot of accuracy and loss over time:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a01d0f13-d0b8-4d78-9ddc-ede5ed402446",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['binary_accuracy', 'loss', 'val_binary_accuracy', 'val_loss'])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history_dict = history.history\n",
    "history_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1f7484c3-3cdf-46d5-b95d-80316f0e6240",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUfUlEQVR4nO3deZyNdf/H8deZGbNhBoNZzDBI9n2LuS2VQmVJNEoMdetO1qQbt52izRYhLbRKaSyVXZRQ3EmppNzZwqAwYx/OXL8/rt8cc8wYs5yZa+ac9/PxOA/nXOc65/ocM3Xevtf3e31shmEYiIiIiLgJL6sLEBEREXElhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiIiIW1G4EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRC/Tu3Zvo6OgcvXb8+PHYbDbXFlTAHDhwAJvNxsKFC/P1uJs2bcJms7Fp0ybHtqz+rPKq5ujoaHr37u3S98yKhQsXYrPZOHDgQL4fWyS3FG5E0rDZbFm6pf3yE8mtrVu3Mn78eM6cOWN1KSJuwcfqAkQKknfffdfp8TvvvMO6devSba9evXqujvP666+TkpKSo9eOHj2aESNG5Or4knW5+Vll1datW5kwYQK9e/emRIkSTs/t3bsXLy/9O1QkOxRuRNJ45JFHnB5/8803rFu3Lt326124cIHAwMAsH6dIkSI5qg/Ax8cHHx/9p5tfcvOzcgU/Pz9Ljy9SGOmfAyLZ1Lp1a2rVqsV3331Hy5YtCQwM5D//+Q8Ay5cv59577yUiIgI/Pz8qV67MpEmTsNvtTu9x/TyO1PkaL7/8MvPnz6dy5cr4+fnRuHFjduzY4fTajObc2Gw2BgwYwLJly6hVqxZ+fn7UrFmT1atXp6t/06ZNNGrUCH9/fypXrsxrr72W5Xk8mzdvplu3bpQvXx4/Pz+ioqJ46qmnuHjxYrrPV6xYMY4cOULnzp0pVqwYZcqUYdiwYen+Ls6cOUPv3r0JDg6mRIkSxMXFZen0zH//+19sNhtvv/12uufWrFmDzWbjs88+A+DgwYM8+eSTVK1alYCAAEJCQujWrVuW5pNkNOcmqzX/+OOP9O7dm0qVKuHv709YWBiPPvoof//9t2Of8ePH88wzzwBQsWJFx6nP1NoymnPzxx9/0K1bN0qVKkVgYCC33XYbn3/+udM+qfOHPvroI5577jkiIyPx9/fnzjvvZN++fTf93DcyZ84catasiZ+fHxEREfTv3z/dZ//999954IEHCAsLw9/fn8jISLp3705iYqJjn3Xr1vGPf/yDEiVKUKxYMapWrer470gkt/TPP5Ec+Pvvv2nfvj3du3fnkUceITQ0FDAnYRYrVoyhQ4dSrFgxvvjiC8aOHUtSUhIvvfTSTd/3gw8+4OzZs/zrX//CZrPx4osv0qVLF/7444+bjiB8/fXXxMfH8+STT1K8eHFeeeUVHnjgAQ4dOkRISAgA33//Pe3atSM8PJwJEyZgt9uZOHEiZcqUydLn/vjjj7lw4QL9+vUjJCSE7du3M2vWLP78808+/vhjp33tdjtt27aladOmvPzyy6xfv56pU6dSuXJl+vXrB4BhGHTq1Imvv/6aJ554gurVq7N06VLi4uJuWkujRo2oVKkSH330Ubr9Fy9eTMmSJWnbti0AO3bsYOvWrXTv3p3IyEgOHDjA3Llzad26Nb/88ku2Rt2yU/O6dev4448/6NOnD2FhYfz888/Mnz+fn3/+mW+++QabzUaXLl347bffWLRoEdOnT6d06dIAN/yZHD9+nObNm3PhwgUGDRpESEgIb7/9Nh07dmTJkiXcf//9Tvs///zzeHl5MWzYMBITE3nxxRfp0aMH3377bZY/c6rx48czYcIE2rRpQ79+/di7dy9z585lx44dbNmyhSJFipCcnEzbtm25fPkyAwcOJCwsjCNHjvDZZ59x5swZgoOD+fnnn7nvvvuoU6cOEydOxM/Pj3379rFly5Zs1ySSIUNEbqh///7G9f+ZtGrVygCMefPmpdv/woUL6bb961//MgIDA41Lly45tsXFxRkVKlRwPN6/f78BGCEhIcapU6cc25cvX24AxqeffurYNm7cuHQ1AYavr6+xb98+x7YffvjBAIxZs2Y5tnXo0MEIDAw0jhw54tj2+++/Gz4+PuneMyMZfb4pU6YYNpvNOHjwoNPnA4yJEyc67Vu/fn2jYcOGjsfLli0zAOPFF190bLt69arRokULAzAWLFiQaT0jR440ihQp4vR3dvnyZaNEiRLGo48+mmnd27ZtMwDjnXfecWzbuHGjARgbN250+ixpf1bZqTmj4y5atMgAjK+++sqx7aWXXjIAY//+/en2r1ChghEXF+d4PGTIEAMwNm/e7Nh29uxZo2LFikZ0dLRht9udPkv16tWNy5cvO/adOXOmARi7d+9Od6y0FixY4FTTiRMnDF9fX+Puu+92HMMwDGP27NkGYLz11luGYRjG999/bwDGxx9/fMP3nj59ugEYJ0+ezLQGkZzSaSmRHPDz86NPnz7ptgcEBDjunz17lr/++osWLVpw4cIFfv3115u+b2xsLCVLlnQ8btGiBWCehriZNm3aULlyZcfjOnXqEBQU5Hit3W5n/fr1dO7cmYiICMd+t9xyC+3bt7/p+4Pz5zt//jx//fUXzZs3xzAMvv/++3T7P/HEE06PW7Ro4fRZVq5ciY+Pj2MkB8Db25uBAwdmqZ7Y2FiuXLlCfHy8Y9vatWs5c+YMsbGxGdZ95coV/v77b2655RZKlCjBzp07s3SsnNSc9riXLl3ir7/+4rbbbgPI9nHTHr9Jkyb84x//cGwrVqwYjz/+OAcOHOCXX35x2r9Pnz74+vo6Hmfndyqt9evXk5yczJAhQ5wmOPft25egoCDHabHg4GDAPDV44cKFDN8rddL08uXL83yytngmhRuRHChXrpzTF0aqn3/+mfvvv5/g4GCCgoIoU6aMYzJy2vkGN1K+fHmnx6lB5/Tp09l+berrU1974sQJLl68yC233JJuv4y2ZeTQoUP07t2bUqVKOebRtGrVCkj/+fz9/dOdWklbD5hzYcLDwylWrJjTflWrVs1SPXXr1qVatWosXrzYsW3x4sWULl2aO+64w7Ht4sWLjB07lqioKPz8/ChdujRlypThzJkzWfq5pJWdmk+dOsXgwYMJDQ0lICCAMmXKULFiRSBrvw83On5Gx0pdwXfw4EGn7bn5nbr+uJD+c/r6+lKpUiXH8xUrVmTo0KG88cYblC5dmrZt2/Lqq686fd7Y2FhiYmL45z//SWhoKN27d+ejjz5S0BGX0ZwbkRxI+y/yVGfOnKFVq1YEBQUxceJEKleujL+/Pzt37mT48OFZ+h+3t7d3htsNw8jT12aF3W7nrrvu4tSpUwwfPpxq1apRtGhRjhw5Qu/evdN9vhvV42qxsbE899xz/PXXXxQvXpwVK1bw0EMPOa0oGzhwIAsWLGDIkCE0a9aM4OBgbDYb3bt3z9Mv1AcffJCtW7fyzDPPUK9ePYoVK0ZKSgrt2rXLty/yvP69yMjUqVPp3bs3y5cvZ+3atQwaNIgpU6bwzTffEBkZSUBAAF999RUbN27k888/Z/Xq1SxevJg77riDtWvX5tvvjrgvhRsRF9m0aRN///038fHxtGzZ0rF9//79FlZ1TdmyZfH3989wpUxWVs/s3r2b3377jbfffptevXo5tq9bty7HNVWoUIENGzZw7tw5p5GQvXv3Zvk9YmNjmTBhAp988gmhoaEkJSXRvXt3p32WLFlCXFwcU6dOdWy7dOlSji6al9WaT58+zYYNG5gwYQJjx451bP/999/TvWd2rjhdoUKFDP9+Uk97VqhQIcvvlR2p77t3714qVark2J6cnMz+/ftp06aN0/61a9emdu3ajB49mq1btxITE8O8efN49tlnAfDy8uLOO+/kzjvvZNq0aUyePJlRo0axcePGdO8lkl06LSXiIqn/2kz7L+Lk5GTmzJljVUlOvL29adOmDcuWLePo0aOO7fv27WPVqlVZej04fz7DMJg5c2aOa7rnnnu4evUqc+fOdWyz2+3MmjUry+9RvXp1ateuzeLFi1m8eDHh4eFO4TK19utHKmbNmpVuWbora87o7wtgxowZ6d6zaNGiAFkKW/fccw/bt29n27Ztjm3nz59n/vz5REdHU6NGjax+lGxp06YNvr6+vPLKK06f6c033yQxMZF7770XgKSkJK5ever02tq1a+Pl5cXly5cB83Td9erVqwfg2EckNzRyI+IizZs3p2TJksTFxTFo0CBsNhvvvvtung7/Z9f48eNZu3YtMTEx9OvXD7vdzuzZs6lVqxa7du3K9LXVqlWjcuXKDBs2jCNHjhAUFMQnn3yS7bkbaXXo0IGYmBhGjBjBgQMHqFGjBvHx8dmejxIbG8vYsWPx9/fnscceS3dF3/vuu493332X4OBgatSowbZt21i/fr1jiXxe1BwUFETLli158cUXuXLlCuXKlWPt2rUZjuQ1bNgQgFGjRtG9e3eKFClChw4dHKEnrREjRrBo0SLat2/PoEGDKFWqFG+//Tb79+/nk08+ybOrGZcpU4aRI0cyYcIE2rVrR8eOHdm7dy9z5syhcePGjrllX3zxBQMGDKBbt27ceuutXL16lXfffRdvb28eeOABACZOnMhXX33FvffeS4UKFThx4gRz5swhMjLSaaK0SE4p3Ii4SEhICJ999hlPP/00o0ePpmTJkjzyyCPceeedjuutWK1hw4asWrWKYcOGMWbMGKKiopg4cSJ79uy56WquIkWK8OmnnzrmT/j7+3P//fczYMAA6tatm6N6vLy8WLFiBUOGDOG9997DZrPRsWNHpk6dSv369bP8PrGxsYwePZoLFy44rZJKNXPmTLy9vXn//fe5dOkSMTExrF+/Pkc/l+zU/MEHHzBw4EBeffVVDMPg7rvvZtWqVU6r1QAaN27MpEmTmDdvHqtXryYlJYX9+/dnGG5CQ0PZunUrw4cPZ9asWVy6dIk6derw6aefOkZP8sr48eMpU6YMs2fP5qmnnqJUqVI8/vjjTJ482XEdprp169K2bVs+/fRTjhw5QmBgIHXr1mXVqlWOlWIdO3bkwIEDvPXWW/z111+ULl2aVq1aMWHCBMdqK5HcsBkF6Z+VImKJzp078/PPP2c4H0REpLDRnBsRD3N9q4Tff/+dlStX0rp1a2sKEhFxMY3ciHiY8PBwR7+jgwcPMnfuXC5fvsz3339PlSpVrC5PRCTXNOdGxMO0a9eORYsWkZCQgJ+fH82aNWPy5MkKNiLiNjRyIyIiIm5Fc25ERETErSjciIiIiFvxuDk3KSkpHD16lOLFi2frkuciIiJiHcMwOHv2LBERETe9WKXHhZujR48SFRVldRkiIiKSA4cPHyYyMjLTfTwu3BQvXhww/3KCgoIsrkZERESyIikpiaioKMf3eGY8LtyknooKCgpSuBERESlksjKlRBOKRURExK0o3IiIiIhbUbgRERERt+Jxc25ERMS17HY7V65csboMcQO+vr43XeadFQo3IiKSI4ZhkJCQwJkzZ6wuRdyEl5cXFStWxNfXN1fvo3AjIiI5khpsypYtS2BgoC6MKrmSepHdY8eOUb58+Vz9PinciIhIttntdkewCQkJsboccRNlypTh6NGjXL16lSJFiuT4fTShWEREsi11jk1gYKDFlYg7ST0dZbfbc/U+CjciIpJjOhUlruSq3yedlnIRux02b4ZjxyA8HFq0AG9vq6sSERHxPBq5cYH4eIiOhttvh4cfNv+Mjja3i4iI+4uOjmbGjBlZ3n/Tpk3YbLY8X2m2cOFCSpQokafHKIgUbnIpPh66doU//3TefuSIuV0BR0Qkc3Y7bNoEixaZf+ZyukWmbDZbprfx48fn6H137NjB448/nuX9mzdvzrFjxwgODs7R8SRzOi2VC3Y7DB4MhpH+OcMAmw2GDIFOnXSKSkQkI/Hx5v9H0/4DMTISZs6ELl1cf7xjx4457i9evJixY8eyd+9ex7ZixYo57huGgd1ux8fn5l+VZcqUyVYdvr6+hIWFZes1knUaucmFzZvTj9ikZRhw+LC5n4iIOLNi5DssLMxxCw4OxmazOR7/+uuvFC9enFWrVtGwYUP8/Pz4+uuv+d///kenTp0IDQ2lWLFiNG7cmPXr1zu97/WnpWw2G2+88Qb3338/gYGBVKlShRUrVjiev/60VOrpozVr1lC9enWKFStGu3btnMLY1atXGTRoECVKlCAkJIThw4cTFxdH586ds/V3MHfuXCpXroyvry9Vq1bl3XffdTxnGAbjx4+nfPny+Pn5ERERwaBBgxzPz5kzhypVquDv709oaChdu3bN1rHzi8JNLqT5nXPJfiIinuJmI99gjnzn5SmqGxkxYgTPP/88e/bsoU6dOpw7d4577rmHDRs28P3339OuXTs6dOjAoUOHMn2fCRMm8OCDD/Ljjz9yzz330KNHD06dOnXD/S9cuMDLL7/Mu+++y1dffcWhQ4cYNmyY4/kXXniB999/nwULFrBlyxaSkpJYtmxZtj7b0qVLGTx4ME8//TQ//fQT//rXv+jTpw8bN24E4JNPPmH69Om89tpr/P777yxbtozatWsD8N///pdBgwYxceJE9u7dy+rVq2nZsmW2jp9vDA+TmJhoAEZiYmKu32vjRsMw/zPM/LZxY64PJSJSoFy8eNH45ZdfjIsXL+bo9QXh/58LFiwwgoOD09S00QCMZcuW3fS1NWvWNGbNmuV4XKFCBWP69OmOx4AxevRox+Nz584ZgLFq1SqnY50+fdpRC2Ds27fP8ZpXX33VCA0NdTwODQ01XnrpJcfjq1evGuXLlzc6deqU5c/YvHlzo2/fvk77dOvWzbjnnnsMwzCMqVOnGrfeequRnJyc7r0++eQTIygoyEhKSrrh8XIrs9+r7Hx/a+QmF1q0MM8N32hZvs0GUVHmfiIick1BHvlu1KiR0+Nz584xbNgwqlevTokSJShWrBh79uy56chNnTp1HPeLFi1KUFAQJ06cuOH+gYGBVK5c2fE4PDzcsX9iYiLHjx+nSZMmjue9vb1p2LBhtj7bnj17iImJcdoWExPDnj17AOjWrRsXL16kUqVK9O3bl6VLl3L16lUA7rrrLipUqEClSpXo2bMn77//PhcuXMjW8fOLwk0ueHubk94gfcBJfTxjhiYTi4hcLzzctfu5UtGiRZ0eDxs2jKVLlzJ58mQ2b97Mrl27qF27NsnJyZm+z/XtA2w2GykpKdna38jovF0eioqKYu/evcyZM4eAgACefPJJWrZsyZUrVyhevDg7d+5k0aJFhIeHM3bsWOrWrVsgG6cq3ORSly6wZAmUK+e8PTLS3J4Xs/1FRAq7wjTyvWXLFnr37s39999P7dq1CQsL48CBA/laQ3BwMKGhoezYscOxzW63s3Pnzmy9T/Xq1dmyZYvTti1btlCjRg3H44CAADp06MArr7zCpk2b2LZtG7t37wbAx8eHNm3a8OKLL/Ljjz9y4MABvvjii1x8sryhpeAu0KWLudxbVygWEcma1JHvrl3NIJN2gKKgjXxXqVKF+Ph4OnTogM1mY8yYMZmOwOSVgQMHMmXKFG655RaqVavGrFmzOH36dLZaFjzzzDM8+OCD1K9fnzZt2vDpp58SHx/vWP21cOFC7HY7TZs2JTAwkPfee4+AgAAqVKjAZ599xh9//EHLli0pWbIkK1euJCUlhapVq+bVR84xhRsX8faG1q2trkJEpPBIHfnO6Do3M2YUnJHvadOm8eijj9K8eXNKly7N8OHDSUpKyvc6hg8fTkJCAr169cLb25vHH3+ctm3b4p2NBNi5c2dmzpzJyy+/zODBg6lYsSILFiyg9f9/gZUoUYLnn3+eoUOHYrfbqV27Np9++ikhISGUKFGC+Ph4xo8fz6VLl6hSpQqLFi2iZs2aefSJc85m5PcJPYslJSURHBxMYmIiQUFBVpcjIlIoXbp0if3791OxYkX8/f1z9V7qzZczKSkpVK9enQcffJBJkyZZXY5LZPZ7lZ3vb43ciIiIpTTynTUHDx5k7dq1tGrVisuXLzN79mz279/Pww8/bHVpBY4mFIuIiBQCXl5eLFy4kMaNGxMTE8Pu3btZv3491atXt7q0AkcjNyIiIoVAVFRUupVOkjGN3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLZ1Lp1a4YMGeJ4HB0dzYwZMzJ9jc1mY9myZbk+tqveJzPjx4+nXr16eXqMvKRwIyIiHqNDhw60a9cuw+c2b96MzWbjxx9/zPb77tixg8cffzy35Tm5UcA4duwY7du3d+mx3I3CjYiIeIzHHnuMdevW8WfaTp3/b8GCBTRq1Ig6depk+33LlClDYGCgK0q8qbCwMPz8/PLlWIWVwo2IiHiM++67jzJlyrBw4UKn7efOnePjjz/mscce4++//+ahhx6iXLlyBAYGUrt2bRYtWpTp+15/Wur333+nZcuW+Pv7U6NGDdatW5fuNcOHD+fWW28lMDCQSpUqMWbMGK5cuQLAwoULmTBhAj/88AM2mw2bzeao+frTUrt37+aOO+4gICCAkJAQHn/8cc6dO+d4vnfv3nTu3JmXX36Z8PBwQkJC6N+/v+NYWZGSksLEiROJjIzEz8+PevXqsXr1asfzycnJDBgwgPDwcPz9/alQoQJTpkwBwDAMxo8fT/ny5fHz8yMiIoJBgwZl+dg5ofYLIiLiEoYBFy5Yc+zAQLDZbr6fj48PvXr1YuHChYwaNQrb/7/o448/xm6389BDD3Hu3DkaNmzI8OHDCQoK4vPPP6dnz55UrlyZJk2a3PQYKSkpdOnShdDQUL799lsSExOd5uekKl68OAsXLiQiIoLdu3fTt29fihcvzr///W9iY2P56aefWL16NevXrwcgODg43XucP3+etm3b0qxZM3bs2MGJEyf45z//yYABA5wC3MaNGwkPD2fjxo3s27eP2NhY6tWrR9++fW/+lwbMnDmTqVOn8tprr1G/fn3eeustOnbsyM8//0yVKlV45ZVXWLFiBR999BHly5fn8OHDHD58GIBPPvmE6dOn8+GHH1KzZk0SEhL44YcfsnTcHDM8TGJiogEYiYmJVpciIlJoXbx40fjll1+MixcvOradO2cYZsTJ/9u5c1mvfc+ePQZgbNy40bGtRYsWxiOPPHLD19x7773G008/7XjcqlUrY/DgwY7HFSpUMKZPn24YhmGsWbPG8PHxMY4cOeJ4ftWqVQZgLF269IbHeOmll4yGDRs6Ho8bN86oW7duuv3Svs/8+fONkiVLGufS/AV8/vnnhpeXl5GQkGAYhmHExcUZFSpUMK5everYp1u3bkZsbOwNa7n+2BEREcZzzz3ntE/jxo2NJ5980jAMwxg4cKBxxx13GCkpKenea+rUqcatt95qJCcn3/B4qTL6vUqVne9vnZYSERGPUq1aNZo3b85bb70FwL59+9i8eTOPPfYYAHa7nUmTJlG7dm1KlSpFsWLFWLNmDYcOHcrS++/Zs4eoqCgiIiIc25o1a5Zuv8WLFxMTE0NYWBjFihVj9OjRWT5G2mPVrVuXokWLOrbFxMSQkpLC3r17Hdtq1qyJt7e343F4eDgnTpzI0jGSkpI4evQoMTExTttjYmLYs2cPYJ762rVrF1WrVmXQoEGsXbvWsV+3bt24ePEilSpVom/fvixdupSrV69m63Nml8KNiIi4RGAgnDtnzS27c3kfe+wxPvnkE86ePcuCBQuoXLkyrVq1AuCll15i5syZDB8+nI0bN7Jr1y7atm1LcnKyy/6utm3bRo8ePbjnnnv47LPP+P777xk1apRLj5FWkSJFnB7bbDZSUlJc9v4NGjRg//79TJo0iYsXL/Lggw/StWtXwOxmvnfvXubMmUNAQABPPvkkLVu2zNacn+zSnBsREXEJmw3SDCAUaA8++CCDBw/mgw8+4J133qFfv36O+TdbtmyhU6dOPPLII4A5h+a3336jRo0aWXrv6tWrc/jwYY4dO0Z4eDgA33zzjdM+W7dupUKFCowaNcqx7eDBg077+Pr6Yrfbb3qshQsXcv78ecfozZYtW/Dy8qJq1apZqvdmgoKCiIiIYMuWLY4AmHqctHOQgoKCiI2NJTY2lq5du9KuXTtOnTpFqVKlCAgIoEOHDnTo0IH+/ftTrVo1du/eTYMGDVxS4/UUbkRExOMUK1aM2NhYRo4cSVJSEr1793Y8V6VKFZYsWcLWrVspWbIk06ZN4/jx41kON23atOHWW28lLi6Ol156iaSkJKcQk3qMQ4cO8eGHH9K4cWM+//xzli5d6rRPdHQ0+/fvZ9euXURGRlK8ePF0S8B79OjBuHHjiIuLY/z48Zw8eZKBAwfSs2dPQkNDc/aXk4FnnnmGcePGUblyZerVq8eCBQvYtWsX77//PgDTpk0jPDyc+vXr4+Xlxccff0xYWBglSpRg4cKF2O12mjZtSmBgIO+99x4BAQFUqFDBZfVdT6elRETEIz322GOcPn2atm3bOs2PGT16NA0aNKBt27a0bt2asLAwOnfunOX39fLyYunSpVy8eJEmTZrwz3/+k+eee85pn44dO/LUU08xYMAA6tWrx9atWxkzZozTPg888ADt2rXj9ttvp0yZMhkuRw8MDGTNmjWcOnWKxo0b07VrV+68805mz56dvb+Mmxg0aBBDhw7l6aefpnbt2qxevZoVK1ZQpUoVwFz59eKLL9KoUSMaN27MgQMHWLlyJV5eXpQoUYLXX3+dmJgY6tSpw/r16/n0008JCQlxaY1p2QzDMPLs3QugpKQkgoODSUxMJCgoyOpyREQKpUuXLrF//34qVqyIv7+/1eWIm8js9yo7398auRERERG3onAjIiIibkXhRkRERNyKwo2IiIi4FYUbERHJMQ9bkyJ5zFW/Two3IiKSbalXvL1gVadMcUupV2hO2yoiJ3QRPxc6fhySkuD/l/2LiLgtb29vSpQo4ehPFBgY6LjCr0hOpKSkcPLkSQIDA/HxyV08UbhxkRUr4OGHoWFD2LTJvAy5iIg7CwsLA8hyA0aRm/Hy8qJ8+fK5DsoKNy5Svz5cvQpffQWrVsE991hdkYhI3rLZbISHh1O2bNk8bYIonsPX1xcvr9zPmFG4cZGoKBg0CF56CUaMgLZtIZenDEVECgVvb+9cz5EQcSVNKHahESOgRAnYvRs++MDqakRERDyTwo0LlSoFI0ea90ePhkuXrK1HRETEEyncuNjAgVCuHBw6BHPnWl2NiIiI51G4cbGAAJgwwbz/7LOQmGhtPSIiIp5G4SYPxMVB9epw6hS8+KLV1YiIiHgWhZs84OMDU6aY96dPh2PHrK1HRETEkyjc5JGOHaF5c7h48dppKhEREcl7Cjd5xGaDF14w77/xBuzda209IiIinkLhJg/94x/QoQPY7TBqlNXViIiIeAbLw82rr75KdHQ0/v7+NG3alO3bt2e6/5kzZ+jfvz/h4eH4+flx6623snLlynyqNvsmTwYvL/jkE/j2W6urERERcX+WhpvFixczdOhQxo0bx86dO6lbty5t27a9YRO25ORk7rrrLg4cOMCSJUvYu3cvr7/+OuXKlcvnyrOuVi1z9RTA8OFgGNbWIyIi4u5shmHd123Tpk1p3Lgxs2fPBsx251FRUQwcOJARI0ak23/evHm89NJL/PrrrxQpUiRHx0xKSiI4OJjExESCgoJyVX9WHT4MVarA5cuwciW0b58vhxUREXEb2fn+tmzkJjk5me+++442bdpcK8bLizZt2rBt27YMX7NixQqaNWtG//79CQ0NpVatWkyePBm73X7D41y+fJmkpCSnW35LbaoJ5uhNJuWKiIhILlkWbv766y/sdjuhoaFO20NDQ0lISMjwNX/88QdLlizBbrezcuVKxowZw9SpU3n22WdveJwpU6YQHBzsuEVFRbn0c2SVmmqKiIjkD8snFGdHSkoKZcuWZf78+TRs2JDY2FhGjRrFvHnzbviakSNHkpiY6LgdPnw4Hyu+plQpM+CAmmqKiIjkJcvCTenSpfH29ub48eNO248fP05YWFiGrwkPD+fWW2/F29vbsa169eokJCSQnJyc4Wv8/PwICgpyulll0CA11RQREclrloUbX19fGjZsyIYNGxzbUlJS2LBhA82aNcvwNTExMezbt4+UlBTHtt9++43w8HB8fX3zvObcSttU87nn1FRTREQkL1h6Wmro0KG8/vrrvP322+zZs4d+/fpx/vx5+vTpA0CvXr0YOXKkY/9+/fpx6tQpBg8ezG+//cbnn3/O5MmT6d+/v1UfIdtSm2r+/Te89JLV1YiIiLgfHysPHhsby8mTJxk7diwJCQnUq1eP1atXOyYZHzp0CC+va/krKiqKNWvW8NRTT1GnTh3KlSvH4MGDGT58uFUfIdtSm2p27gzTpkH//hAebnVVIiIi7sPS69xYwYrr3FzPMMzWDFu3wr/+BZnMhxYREREKyXVuPJnNBs8/b95XU00RERHXUrixSIsW15pqjh5tdTUiIiLuQ+HGQqlNNZcsUVNNERERV1G4sZCaaoqIiLiewo3FJkwAPz/48ktYvdrqakRERAo/hRuLRUXBwIHmfTXVFBERyT2FmwJg5EgIDlZTTREREVdQuCkASpUyAw7AmDFw+bK19YiIiBRmCjcFRGpTzYMH1VRTREQkNxRuCoi0TTWffVZNNUVERHJK4aYAUVNNERGR3FO4KUB8fMwL+4HZVPPYMWvrERERKYwUbgqYTp2gWTO4ePHaaSoRERHJOoWbAsZmgxdeMO+/8Qb89pu19YiIiBQ2CjcFUNqmmqNGWV2NiIhI4aJwU0CpqaaIiEjOKNwUULVqQa9e5n011RQREck6hZsCTE01RUREsk/hpgArX/5aU80RIyAlxdp6RERECgOFmwIutanmjz+qqaaIiEhWKNwUcGmbao4eraaaIiIiN6NwUwioqaaIiEjWKdwUAgEBMH68eV9NNUVERDKncFNI9O4N1aqpqaaIiMjNKNwUEj4+MGWKeX/6dDXVFBERuRGFm0IktanmhQswcWLeHMNuh02bYNEi80+7PW+OIyIiklcUbgqRtE01X3/d9U014+MhOhpuvx0eftj8Mzra3C4iIlJYKNwUMi1awH33ub6pZnw8dO0Kf/7pvP3IEXO7Ao6IiBQWCjeF0JQp5ijOkiWwfXvu389uh8GDM+5flbptyBCdohIRkcJB4aYQqlUL4uLM+65oqrl5c/oRm7QMAw4fNvcTEREp6BRuCqnUppqbNsGaNbl7r6yuvNIKLRERKQwUbgqptE01hw/PXVPN8HDX7iciImIlhZtCzFVNNVu0gMhIcx5PRmw2iIoy9xMRESnoFG4KsVKlYMQI835ummp6e8PMmeb96wNO6uMZM8z9RERECjqFm0Ju0CCIiMh9U80uXczVV+XKOW+PjDS3d+mSuzpFRETyi80wcrvWpnBJSkoiODiYxMREgoKCrC7HJd54A/r2hZAQ+N//zFNVOWW3m6uijh0z59i0aKERGxERsV52vr81cuMG0jbVfPnl3L2Xtze0bg0PPWT+qWAjIiKFjcKNG0jbVHPaNC3ZFhERz6Zw4ybyo6mmiIhIYaBw4ybyuqmmiIhIYaFw40byqqmmiIhIYaJw42Zc3VRTRESksFG4cTOubqopIiJS2CjcuCFXNtUUEREpbBRu3FD58jBggHk/t001RUREChuFGzflqqaaIiIihY3CjZsKCbnWVHPMmJw31RQRESlsFG7cWGpTzQMHYN48q6sRERHJHwo3biww0JxcDDBpEiQmWluPiIhIflC4cXOubKopIiJSGCjcuDkfH5g82byvppoiIuIJFG48QOfOcNttaqopIiKeQeHGA6ippoiIeBKFGw/RsuW1ppqjR1tdjYiISN5RuPEgqU01P/5YTTVFRMR9Kdx4kFq1oFcv876aaoqIiLtSuPEwEyeqqaaIiLg3hRsPk7ap5ogRaqopIiLuR+HGA6U21fzhB1i0yOpqREREXEvhxgOlbao5erSaaoqIiHtRuPFQaqopIiLuSuHGQwUGwvjx5n011RQREXeicOPB+vSBqlXVVFNERNyLwo0H8/ExL+wHZlPNhARr6xEREXEFhRsPp6aaIiLibhRuPFzapprz56uppoiIFH4KN6KmmiIi4lYKRLh59dVXiY6Oxt/fn6ZNm7I9k66OCxcuxGazOd38/f3zsVr3NHnytaaaO3ZYXY2IiEjOWR5uFi9ezNChQxk3bhw7d+6kbt26tG3blhMnTtzwNUFBQRw7dsxxO3jwYD5W7J5q11ZTTRERcQ+Wh5tp06bRt29f+vTpQ40aNZg3bx6BgYG89dZbN3yNzWYjLCzMcQsNDc3Hit1XalPNjRth7VqrqxEREckZS8NNcnIy3333HW3atHFs8/Lyok2bNmzbtu2Grzt37hwVKlQgKiqKTp068fPPP+dHuW4vbVPNp582V1CJiIgUNpaGm7/++gu73Z5u5CU0NJSEG1x0pWrVqrz11lssX76c9957j5SUFJo3b86ff/6Z4f6XL18mKSnJ6SY3NnIklCkDP/8MPXuqa7iIiBQ+lp+Wyq5mzZrRq1cv6tWrR6tWrYiPj6dMmTK89tprGe4/ZcoUgoODHbeoqKh8rrhwCQmB+Hjw9TX/1OopEREpbCwNN6VLl8bb25vjx487bT9+/DhhYWFZeo8iRYpQv3599u3bl+HzI0eOJDEx0XE7fPhwrut2d//4B7zxhnl/yhR4+21r6xEREckOS8ONr68vDRs2ZMOGDY5tKSkpbNiwgWbNmmXpPex2O7t37yY8PDzD5/38/AgKCnK6yc317AmjRpn3+/aFzZutrUdERCSrLD8tNXToUF5//XXefvtt9uzZQ79+/Th//jx9+vQBoFevXowcOdKx/8SJE1m7di1//PEHO3fu5JFHHuHgwYP885//tOojuK2JE6FrV7hyBe6/H/73P6srEhERuTkfqwuIjY3l5MmTjB07loSEBOrVq8fq1asdk4wPHTqEl9e1DHb69Gn69u1LQkICJUuWpGHDhmzdupUaNWpY9RHclpeXeUrq4EHzwn733QfbtkGJElZXJiIicmM2w/Csy7UlJSURHBxMYmKiTlFl0bFj0KQJ/PkntGkDK1dCkSJWVyUiIp4kO9/flp+WkoIvPBw+/RSKFoX162HgQF3BWERECi6FG8mSevVg0SKz/9Rrr8Err1hdkYiISMYUbiTLOnSAl1827w8dCp9/bm09IiIiGVG4kWx56ilzaXhKCnTvDj/+aHVFIiIizhRuJFtsNnj1VbjjDjh3zhzNuUGnDBEREUso3Ei2FSkCS5bArbfCoUPQuTNcvGh1VSIiIiaFG8mRkiXhs8+gVCn49lvo00crqEREpGBQuJEcq1LFbK5ZpAgsXgzjx1tdkYiIiMKN5FKrVubScDDbNbz/vrX1iIiIKNxIrvXpA//+t3n/0Udh61Zr6xEREc+mcCMuMWWKObE4Odn888ABiwsSERGPpXAjLuHlBe+9B/Xrw8mTZpPNpCSrqxIREU+kcCMuU7So2YMqIgJ+/hliY+HqVaurEhERT6NwIy5VrhysWAEBAbB6tdmmQUREJD8p3IjLNWxonqICmDXLvKKxiIhIflG4kTzRpYs5yRhg8GBYs8baekRExHMo3EieGT4cevcGux0efBB++cXqikRExBMo3EiesdnMC/y1bGmunLrvPnMllYiISF5SuJE85esLn3wClSvD/v3mNXAuXbK6KhERcWc5CjeHDx/mzz//dDzevn07Q4YMYf78+S4rTNxH6dJmk83gYPPqxX37qsmmiIjknRyFm4cffpiNGzcCkJCQwF133cX27dsZNWoUEydOdGmB4h6qVYMlS8Db21xJNXmy1RWJiIi7ylG4+emnn2jSpAkAH330EbVq1WLr1q28//77LFy40JX1iRtp0wbmzDHvjx4NH31kbT0iIuKechRurly5gp+fHwDr16+nY8eOAFSrVo1jx465rjpxO48/Dk89Zd6Pi4Pt262tR0RE3E+Owk3NmjWZN28emzdvZt26dbRr1w6Ao0ePEhIS4tICxf289BLce685sbhjRzh0yOqKRETEneQo3Lzwwgu89tprtG7dmoceeoi6desCsGLFCsfpKpEb8faGRYugTh04fhw6dICzZ62uSkRE3IXNMHK2bsVut5OUlETJkiUd2w4cOEBgYCBly5Z1WYGulpSURHBwMImJiQQFBVldjkc7dAiaNLkWcJYuNYOPiIjI9bLz/Z2jkZuLFy9y+fJlR7A5ePAgM2bMYO/evQU62EjBUr48LF8O/v5mN/F//9vqikRExB3kKNx06tSJd955B4AzZ87QtGlTpk6dSufOnZk7d65LCxT31rQpvP22eX/aNNClkkREJLdyFG527txJixYtAFiyZAmhoaEcPHiQd955h1deecWlBYr7e/BBSL08Uv/+sGGDtfWIiEjhlqNwc+HCBYoXLw7A2rVr6dKlC15eXtx2220cPHjQpQWKZxg9Gnr0gKtXoWtX2Ls3d+9nt8OmTebE5U2bzMciIuIZchRubrnlFpYtW8bhw4dZs2YNd999NwAnTpzQJF3JEZsN3ngDmjeHM2fMJpt//52z94qPh+houP12ePhh88/oaHO7iIi4vxyFm7FjxzJs2DCio6Np0qQJzZo1A8xRnPr167u0QPEc/v7miqnoaNi3Dx54AJKTs/ce8fHmyE+a1mcAHDliblfAERFxfzleCp6QkMCxY8eoW7cuXl5mRtq+fTtBQUFUq1bNpUW6kpaCF3w//wzNmpnXvunTB9580xzZuRm73QxG1webVDYbREaa3cm15FxEpHDJ86XgAGFhYdSvX5+jR486OoQ3adKkQAcbKRxq1jT7Tnl5wYIF5hWNs2Lz5hsHGzA7kR8+bO4nIiLuK0fhJiUlhYkTJxIcHEyFChWoUKECJUqUYNKkSaSkpLi6RvFA7drBzJnm/REjzNNVN5PVtmZqfyYi4t58cvKiUaNG8eabb/L8888TExMDwNdff8348eO5dOkSzz33nEuLFM80YIC5amr2bHjkEXPEpUGDG+8fHp61983qfiIiUjjlaM5NREQE8+bNc3QDT7V8+XKefPJJjhw54rICXU1zbgqXq1fNlVNr1kBEhNlFvFy5jPdNnXNz5Ih5Cup6mnMjIlJ45fmcm1OnTmU4t6ZatWqcOnUqJ28pkiEfH1i8GGrUgKNHzS7i589nvK+397VTWddPQE59PGOGgo2IiLvLUbipW7cus2fPTrd99uzZ1KlTJ9dFiaQVHAyffQZlysDOndCzJ9xoaleXLrBkSfrRnchIc3uXLnlfr4iIWCtHp6W+/PJL7r33XsqXL++4xs22bds4fPgwK1eudLRmKIh0Wqrw2rrVvCBfcrI5yXjKlBvva7ebc3SOHTPn2LRooREbEZHCLM9PS7Vq1YrffvuN+++/nzNnznDmzBm6dOnCzz//zLvvvpujokVupnlzeOst8/7zz5vLxG/E2xtat4aHHjL/VLAREfEcOb6IX0Z++OEHGjRogL0AN/LRyE3hN3YsTJoERYrAunXQqpXVFYmISF7Ll4v4iVhl/Hizk/iVK+Ycmn37rK5IREQKEoUbKXS8vGDhQmjSBE6dMpeKnz5tdVUiIlJQKNxIoRQQAMuXQ1SUeaG/bt3MkRwREZFsXaG4y03W0Z45cyY3tYhkS1iYuUQ8JgY2bDCvaDxvXtaabIqIiPvKVrgJDg6+6fO9evXKVUEi2VGnDixaZF7cb/58qFYNnnrK6qpERMRKLl0tVRhotZR7mj4dhg41R22WL4cOHayuSEREXEmrpcTjDBkC//qX2VPqoYfghx+srkhERKyicCNuwWaDWbPgzjvN3lMdOkBCgtVViYiIFRRuxG0UKQIffwxVq8Lhw9CpE1y8aHVVIiKS3xRuxK2ULGmuoCpVCrZvhx494MIFq6sSEZH8pHAjbueWW2DpUnMkZ+lSaNjQ7CYuIiKeQeFG3FLLlrB6NUREwK+/QtOmZrPNAtz2TEREXEThRtzWHXfAjz/CAw/A1aswcqQ54fjQIasrExGRvKRwI24tJMScZPzWW1CsGHz5pXnhvw8+sLoyERHJKwo34vZsNujTB3btgttug8REc6Jxjx6gjiEiIu5H4UY8RuXKsHkzjB8P3t7m6E3duvDVV1ZXJiIirqRwIx7FxwfGjYOvv4ZKlcz5N61bm/NxkpOtrk5ERFxB4UY80m23maepHn3UbNnw/PPQrJm5skpERAo3hRvxWMWLw5tvwiefmBf927kTGjSAefPMwCMiIoWTwo14vC5dYPduuOsus11Dv37QsSOcOGF1ZSIikhMKNyKYF/tbvRqmTwc/P7OFQ+3a8PnnVlcmIiLZpXAj8v+8vGDIENixwww2J07AfffBk0+qP5WISGGicCNyndq1zaabTz1lPp47V/2pREQKE4UbkQz4+8O0abB27bX+VLfdBi+8oP5UIiIFncKNSCbuuutaf6orV2DECPWnEhEp6BRuRG7iRv2pFi2yujIREclIgQg3r776KtHR0fj7+9O0aVO2b9+epdd9+OGH2Gw2OnfunLcFisfLqD/Vww/DI4+oP5WISEFjebhZvHgxQ4cOZdy4cezcuZO6devStm1bTtzkIiMHDhxg2LBhtGjRIp8qFUnfn+r999WfSkSkoLE83EybNo2+ffvSp08fatSowbx58wgMDOStt9664Wvsdjs9evRgwoQJVKpUKR+rFVF/KhGRgs7ScJOcnMx3331HmzZtHNu8vLxo06YN27Ztu+HrJk6cSNmyZXnsscdueozLly+TlJTkdBNxhYz6UzVvrv5UIiJWszTc/PXXX9jtdkJDQ522h4aGkpCQkOFrvv76a958801ef/31LB1jypQpBAcHO25RUVG5rlsk1fX9qb77Tv2pRESsZvlpqew4e/YsPXv25PXXX6d06dJZes3IkSNJTEx03A4fPpzHVYon6tLFXDLepo36U4mIWM3HyoOXLl0ab29vjh8/7rT9+PHjhIWFpdv/f//7HwcOHKBDhw6ObSkpKQD4+Piwd+9eKleu7PQaPz8//Pz88qB6EWflysGaNfDKK+b1cFL7U731Ftx7r9XViYh4DktHbnx9fWnYsCEbNmxwbEtJSWHDhg00a9Ys3f7VqlVj9+7d7Nq1y3Hr2LEjt99+O7t27dIpJ7Fc2v5UtWpd60/Vv7/6U4mI5BdLR24Ahg4dSlxcHI0aNaJJkybMmDGD8+fP06dPHwB69epFuXLlmDJlCv7+/tSqVcvp9SVKlABIt13ESrVrmwHnP/8xO43PmQNffGEuHW/QwOrqRETcm+XhJjY2lpMnTzJ27FgSEhKoV68eq1evdkwyPnToEF5ehWpqkAhwrT9V+/bQu/e1/lSTJsGwYeZ1ckRExPVshuFZazqSkpIIDg4mMTGRoKAgq8sRD/H33/D44xAfbz5u1QreeQfKl7e2LhGRwiI7398aEhHJByEhsGRJ9vtT2e2waZO536ZN6kguIpIVCjci+SS7/ani4yE6Gm6/3dzv9tvNx6mjPyIikjGFG5F8lpX+VPHx0LUr/Pmn82uPHDG3K+CIiNyYwo2IBVL7U23e7Nyf6j//MS8COHhwxlc4Tt02ZIhOUYmI3IjCjYiFmjVz7k81ZQrUq5d+xCYtw4DDh81gJCIi6SnciFjs+v5Uv/2WtdcdO5a3dYmIFFYKNyIFRGp/qoYNs7Z/eHje1iMiUlgp3IgUIOXKwbZtEBx8431sNoiKghYt8q8uEZHCROFGpIApUsS8Hs6NGAbMmKErHIuI3IjCjUgB1KWLOQenXLn0zwUHw+7dZlNOERFJT+0XRAowu91cFfX77/Df/8LKlddWUvn6Qo8e5rLxunWtrVNEJK9l5/tb4UakELlyxbyA3/Tp8O2317bffjs89RTcey+oz6yIuCP1lhJxU0WKQGwsfPONOfE4Ntace7NxI3TsCFWrwuzZcO6c1ZWKiFhH4UakkLrtNvjwQ/jjD/j3v6FECdi3DwYOhMhIeOYZOHjQ6ipFRPKfwo1IIVe+PLzwgjkX59VX4dZbzaacL79stnbo1g22bs24nYOIiDtSuBFxE0WLwpNPwp498Pnn0KYNpKTAkiUQEwNNm8IHH5jzdkRE3JnCjYib8fKCe+6BdevMJeOPPQZ+frBjh7m6Kjra7GH1999WVyoikjcUbkTcWK1a8MYbZqPNSZMgLAyOHjW7j0dFwRNPmCM9IiLuROFGxAOUKQOjR8OBA/DOO1C/Ply8CK+9BjVqQPv2sGaN5uWIiHtQuBHxIH5+0LMnfPcdfPkl3H+/2atq9Wpo1w5q1jQDz4ULVlcqIpJzCjciHshmg5YtzQsC7tsHQ4ZA8eLmKaonnjBPWf3nP3DkiNWViohkn8KNiIerVMm84vGff5p/VqwIp06Zk46jo81JyDt2WF2liEjWKdyICABBQeYIzu+/myM6LVvC1avm8vEmTeAf/zCXlV+9anWlIiKZU7gRESfe3uZcnC+/NOfm9Oxptn3YssW8IOAtt8DUqXDmjNWViohkTOFGRG6oQQNzddXBgzBmDJQubd4fNsxs8TBokDlnR0SkIFG4EZGbCg+HiRPh0CHzujm1asH58zBrltnuoWNHs3mnlpKLSEGgcCMiWRYQYF7x+McfzSsg33uvGWg+/RTuuMO8fs7ChXDpktWViognU7gRkWyz2czeVZ99Br/+ava0CgyEH36APn2gQgUYPx6OH7e6UhHxRAo3IpIrVaua3cj//NPsTh4ZCSdOwIQJZsfyPn3M0CMikl8UbkTEJUqWhH//G/74AxYvhttug+Rk8zRVvXrmaaulS3XKSkTynsKNiLhUkSLw4IOwbZt5697dXF6+cSN06WKuuOraFd57D06ftrpaEXFHNsPwrPUNSUlJBAcHk5iYSFBQkNXliHiEw4fNU1fvvefc0sHbG1q3hk6dzFv58paVKCIFXHa+vxVuRCTfGIZ5YcBly2D5cvjpJ+fnGzQwQ07nzlC7tjlxWUQEFG4ypXAjkv/sdti8GY4dM6+Z06KFOWqzb58ZcpYvh6+/dr5OTsWK14JOTAz4+FhWvogUAAo3mVC4Eclf8fEweLC5mipVZCTMnGnOwUl14oS5tHz5cli71nnicUgIdOhghp277zaXnYuIZ1G4yYTCjUj+iY83Jw9f/3+Z1NNNS5Y4B5xU58+bAWfZMjPwnDp17bmAADPgdOoE990HZcrkWfkiUoAo3GRC4UYkf9jtEB3tPGKTls1mjuDs32+eorqRq1fNU1ap83QOHLj2nJeX2a089fRVpUquq19EChaFm0wo3Ijkj02b4Pbbb77fxo3miqmsMAyz9UNq0Pn+e+fna9e+FnQaNNCEZBF3kp3vb13nRkTyxLFjrt0PzLBSty6MGwc7d5qjODNnmiHK2xt274Znn4VGjcwWEAMHwvr1cOVKjj6CiBRSCjcikifCw127X0YqVIBBg+CLL8wJye+8Y87hCQw0r60zezbcdReULQuPPAIffwxnz+b8eCJSOOi0lIjkidQ5N0eOpJ9QDFmfc5MTFy/Chg3m6asVK+DkyWvP+fqaTT87dYKOHSEszLXHFpG8oTk3mVC4Eck/qaulwDng3Gy1lCvZ7fDNN2bQWbbMvLZO2jpuu82co9Opk9kEVEQKJoWbTCjciOSvjK5zExUFM2bkfbC5nmHAnj3Xgs6OHc7PV6tmBp3OnaFxY3M1logUDAo3mVC4Ecl/N7pCsdWOHDFPWy1bZq7aSjvxODzcPG3VqZPZ0dzPz7IyRQSFm0wp3IhIRhITYdUqM+isXOk88bh4cWjf3hzRad8eSpSwqEgRD6ZwkwmFGxG5mcuXzev0pF5PJ+1ydR8faNXK7HfVpIl501WSRfKewk0mFG5EJDtSUuC//702T2fPnvT7VKx4Leg0bQr166v/lYirKdxkQuFGRHLjt9/MCwNu327eMgo73t7m1ZKbNr0WeqpXLxjzjEQKK4WbTCjciIgrJSaaIzvbt8O335q3hIT0+xUrZl45OXV0p0kTKFdOLSJEskrhJhMKNyKSlwzDXIX17bfXRnd27DA7nV8vPNx5dKdxY9D/lkQypnCTCYUbEclvdrt5+ip1dGf7drMPlt3uvJ/NZl5rJ+3oTu3a5lWVRTydwk0mFG5EpCC4cMFs/pk6uvPtt2Yj0Ov5+ZkdztNOWK5USaezxPMo3GRC4UZECqoTJ8xTWGlPaZ0+nX6/UqWcR3caN9ZydHF/CjeZULgRkcLCMMxeWKlBZ/t2+P578zo816tUKf1y9ICA/K9ZJK8o3GRC4UZECrPkZPjxR+f5O7/+mn4/H5/0y9GrVdNydCm8FG4yoXAjIu7mzJlry9FTQ09Gy9GLF3dejt64sZajS+GhcJMJhRsRyamC2gD0eoZhdmFPO7rz3/9mvBw9ONi8wGCNGs5/VqigruhSsCjcZELhRkRyIj4eBg82Q0OqyEiYORO6dLGurqyy2+GXX5xHd376Kf1y9FQBAeZprOtDT+XKUKRI/tYuAgo3mVK4EZHsio+Hrl3NEZG0Uk/nLFlSOALO9S5fht9/N0PPnj3X/ty715zbk5EiRaBKlfSjPbfeqgnMkrcUbjKhcCMi2WG3Q3S084hNWjabOYKzf3/BPEWVE1evmp/n+tCzZ0/Gp7bA/HuoVCl96KlWTVddFtdQuMmEwo2IZMemTXD77Tffb+NGaN06r6uxVkqKGfKuDz2//JLx9XhSRUZmPK+ndOn8q10Kv+x8f/vkU00iIoXSsWOu3a8w8/KC8uXNW7t217YbhnkBwoxGeo4dMwPRn3/CunXO71emTMahJyJCK7gkdxRuREQyER7u2v3ckc0GoaHm7fpRrtOnzevwXB98DhyAkyfN21dfOb8mKCjj0BMdrRVckjU6LSUikonUOTdHjqSfUAzuOecmP5w/b05c/uUX5+Dzv/9lvoKratX0weeWW7SCyxNozk0mFG5EJLtSV0uBc8Ap7KulCqLUFVypp7XSruDKqO0EmKEyIsK8IGFmt8DA/P0s4loKN5lQuBGRnMjoOjdRUTBjhoJNfrDbM17B9csvN17Bdb0SJZzDTmRk+gBUurROfRVUCjeZULgRkZwqLFco9iSGAUePmqHzyJEb37IagIoUydookL9/3n4uSa/QhZtXX32Vl156iYSEBOrWrcusWbNo0qRJhvvGx8czefJk9u3bx5UrV6hSpQpPP/00PXv2zNKxFG5ERDyLYUBiYubh58gRc8VXVr8RQ0JuHoBCQrTqy5UK1VLwxYsXM3ToUObNm0fTpk2ZMWMGbdu2Ze/evZQtWzbd/qVKlWLUqFFUq1YNX19fPvvsM/r06UPZsmVp27atBZ9AREQKMpvNPCVVogTUrHnj/a5cMUflbhR+UkeHLl2Cv/82bz/+eOP38/NLPwp0/amwiAjw9XX1JxbLR26aNm1K48aNmT17NgApKSlERUUxcOBARowYkaX3aNCgAffeey+TJk266b4auRERkZwyDHN5+81GgU6ezPp7lilzLeyUKQOlSmV+CwryzBGhQjNyk5yczHfffcfIkSMd27y8vGjTpg3btm276esNw+CLL75g7969vPDCC3lZqoiICDbbtZBRu/aN97t8OeNRoLRzg44eNfdLvd7Prl1Zq8Hb++YBKO0tJMT8MzjYcyZLWxpu/vrrL+x2O6GhoU7bQ0ND+fXXX2/4usTERMqVK8fly5fx9vZmzpw53HXXXRnue/nyZS6nWT+YlJTkmuJFRERuwM/PvD5SdPSN9zEM89RW2rDz999w6lT6W+r2ixfNie2pgSg7bDYoWTJrQSjtrUQJ8LF8Ekv2FLJyTcWLF2fXrl2cO3eODRs2MHToUCpVqkTrDBq7TJkyhQkTJuR/kSIiIpmw2cyl56VLQ926WXvNxYvmabGMAlDaEHT97dw5M0ylPs6u4OCbh6C0t9KlzVNsVrF0zk1ycjKBgYEsWbKEzp07O7bHxcVx5swZli9fnqX3+ec//8nhw4dZs2ZNuucyGrmJiorSnBsREfEYycnXQtGNAlBGt8TEnB2vfn3YudO1n6HQzLnx9fWlYcOGbNiwwRFuUlJS2LBhAwMGDMjy+6SkpDgFmLT8/Pzw8/NzRbkiIiKFkq/vtf5f2XHlCpw5c/MQdH1gsrrju+WnpYYOHUpcXByNGjWiSZMmzJgxg/Pnz9OnTx8AevXqRbly5ZgyZQpgnmZq1KgRlStX5vLly6xcuZJ3332XuXPnWvkxREQKDV2MULKqSBHz9FJ2TzFZfQU9y8NNbGwsJ0+eZOzYsSQkJFCvXj1Wr17tmGR86NAhvNJM7z5//jxPPvkkf/75JwEBAVSrVo333nuP2NhYqz6CiEihkVEbichImDlTbSTEdaxeqm75dW7ym65zIyKeKrUB6PX/11cDUCkMsvP97SEr3kVEPJvdbo7YZPTP2dRtQ4aY+4kUdgo3IiIeYPNm51NR1zMMOHzY3E+ksFO4ERHxAMeOuXY/kYJM4UZExAOEh7t2P5GCTOFGRMQDtGhhroq60SoWmw2iosz9RAo7hRsREQ/g7W0u94b0ASf18YwZut6NuAeFGxERD9Gli7ncu1w55+2RkVoGLu7F8ov4iYhI/unSBTp10hWKxb0p3IiIeBhvb2jd2uoqRPKOTkuJiIiIW1G4EREREbeicCMiIiJuRXNuRESk0LLbNTla0lO4ERGRQik+3mwGmrZnVmSkeT0fLWv3bDotJSIihU58PHTtmr4Z6JEj5vb4eGvqkoJB4UZERAoVu90csTGM9M+lbhsyxNxPPJPCjYiIFCqbN6cfsUnLMODwYXM/8UwKNyIiUqgcO+ba/cT9KNyIiEihEh7u2v3E/SjciIhIodKihbkq6vru5qlsNoiKMvcTz6RwIyIihYq3t7ncG9IHnNTHM2boejeeTOFGREQKnS5dYMkSKFfOeXtkpLld17nxbLqIn4iIFEpdukCnTrpCsaSncCMiIoWWtze0bm11FVLQ6LSUiIiIuBWFGxEREXErOi0lIiJiMXU3dy2FGxEREQupu7nr6bSUiIiIRdTdPG8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhE3c3zhi7iJyIiYiF1N3c9hRsRERGLuUt384LSRkLhRkRERHKtILWR0JwbERERyZWC1kZC4UZERERyrCC2kVC4ERERkRwriG0kFG5EREQkxwpiGwmFGxEREcmxgthGQuFGREREcqwgtpFQuBEREZEcK4htJBRuREREJFcKWhsJXcRPREREcq0gtZFQuBERERGXKChtJHRaSkRERNyKwo2IiIi4FYUbERERcSsKNyIiIuJWFG5ERETErSjciIiIiFtRuBERERG3onAjIiIibkXhRkRERNyKx12h2DAMAJKSkiyuRERERLIq9Xs79Xs8Mx4Xbs6ePQtAVFSUxZWIiIhIdp09e5bg4OBM97EZWYlAbiQlJYWjR49SvHhxbNf3ZhfATMdRUVEcPnyYoKAgq8vxePp5FCz6eRQ8+pkULHn18zAMg7NnzxIREYGXV+azajxu5MbLy4vIyEiryygUgoKC9D+KAkQ/j4JFP4+CRz+TgiUvfh43G7FJpQnFIiIi4lYUbkRERMStKNxIOn5+fowbNw4/Pz+rSxH08yho9PMoePQzKVgKws/D4yYUi4iIiHvTyI2IiIi4FYUbERERcSsKNyIiIuJWFG5ERETErSjciMOUKVNo3LgxxYsXp2zZsnTu3Jm9e/daXZYAzz//PDabjSFDhlhdikc7cuQIjzzyCCEhIQQEBFC7dm3++9//Wl2WR7Lb7YwZM4aKFSsSEBBA5cqVmTRpUpb6DknuffXVV3To0IGIiAhsNhvLli1zet4wDMaOHUt4eDgBAQG0adOG33//Pd/qU7gRhy+//JL+/fvzzTffsG7dOq5cucLdd9/N+fPnrS7No+3YsYPXXnuNOnXqWF2KRzt9+jQxMTEUKVKEVatW8csvvzB16lRKlixpdWke6YUXXmDu3LnMnj2bPXv28MILL/Diiy8ya9Ysq0vzCOfPn6du3bq8+uqrGT7/4osv8sorrzBv3jy+/fZbihYtStu2bbl06VK+1Kel4HJDJ0+epGzZsnz55Ze0bNnS6nI80rlz52jQoAFz5szh2WefpV69esyYMcPqsjzSiBEj2LJlC5s3b7a6FAHuu+8+QkNDefPNNx3bHnjgAQICAnjvvfcsrMzz2Gw2li5dSufOnQFz1CYiIoKnn36aYcOGAZCYmEhoaCgLFy6ke/fueV6TRm7khhITEwEoVaqUxZV4rv79+3PvvffSpk0bq0vxeCtWrKBRo0Z069aNsmXLUr9+fV5//XWry/JYzZs3Z8OGDfz2228A/PDDD3z99de0b9/e4spk//79JCQkOP1/Kzg4mKZNm7Jt27Z8qcHjGmdK1qSkpDBkyBBiYmKoVauW1eV4pA8//JCdO3eyY8cOq0sR4I8//mDu3LkMHTqU//znP+zYsYNBgwbh6+tLXFyc1eV5nBEjRpCUlES1atXw9vbGbrfz3HPP0aNHD6tL83gJCQkAhIaGOm0PDQ11PJfXFG4kQ/379+enn37i66+/troUj3T48GEGDx7MunXr8Pf3t7ocwQz8jRo1YvLkyQDUr1+fn376iXnz5incWOCjjz7i/fff54MPPqBmzZrs2rWLIUOGEBERoZ+H6LSUpDdgwAA+++wzNm7cSGRkpNXleKTvvvuOEydO0KBBA3x8fPDx8eHLL7/klVdewcfHB7vdbnWJHic8PJwaNWo4batevTqHDh2yqCLP9swzzzBixAi6d+9O7dq16dmzJ0899RRTpkyxujSPFxYWBsDx48edth8/ftzxXF5TuBEHwzAYMGAAS5cu5YsvvqBixYpWl+Sx7rzzTnbv3s2uXbsct0aNGtGjRw927dqFt7e31SV6nJiYmHSXRvjtt9+oUKGCRRV5tgsXLuDl5fwV5u3tTUpKikUVSaqKFSsSFhbGhg0bHNuSkpL49ttvadasWb7UoNNS4tC/f38++OADli9fTvHixR3nRoODgwkICLC4Os9SvHjxdHOdihYtSkhIiOZAWeSpp56iefPmTJ48mQcffJDt27czf/585s+fb3VpHqlDhw4899xzlC9fnpo1a/L9998zbdo0Hn30UatL8wjnzp1j3759jsf79+9n165dlCpVivLlyzNkyBCeffZZqlSpQsWKFRkzZgwRERGOFVV5zhD5f0CGtwULFlhdmhiG0apVK2Pw4MFWl+HRPv30U6NWrVqGn5+fUa1aNWP+/PlWl+SxkpKSjMGDBxvly5c3/P39jUqVKhmjRo0yLl++bHVpHmHjxo0Zfl/ExcUZhmEYKSkpxpgxY4zQ0FDDz8/PuPPOO429e/fmW326zo2IiIi4Fc25EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiEey2WwsW7bM6jJEJA8o3IhIvuvduzc2my3drV27dlaXJiJuQL2lRMQS7dq1Y8GCBU7b/Pz8LKpGRNyJRm5ExBJ+fn6EhYU53UqWLAmYp4zmzp1L+/btCQgIoFKlSixZssTp9bt37+aOO+4gICCAkJAQHn/8cc6dO+e0z1tvvUXNmjXx8/MjPDycAQMGOD3/119/cf/99xMYGEiVKlVYsWKF47nTp0/To0cPypQpQ0BAAFWqVEkXxkSkYFK4EZECacyYMTzwwAP88MMP9OjRg+7du7Nnzx4Azp8/T9u2bSlZsiQ7duzg448/Zv369U7hZe7cufTv35/HH3+c3bt3s2LFCm655RanY0yYMIEHH3yQH3/8kXvuuYcePXpw6tQpx/F/+eUXVq1axZ49e5g7dy6lS5fOv78AEcm5fGvRKSLy/+Li4gxvb2+jaNGiTrfnnnvOMAyzQ/0TTzzh9JqmTZsa/fr1MwzDMObPn2+ULFnSOHfunOP5zz//3PDy8jISEhIMwzCMiIgIY9SoUTesATBGjx7teHzu3DkDMFatWmUYhmF06NDB6NOnj2s+sIjkK825ERFL3H777cydO9dpW6lSpRz3mzVr5vRcs2bN2LVrFwB79uyhbt26FC1a1PF8TEwMKSkp7N27F5vNxtGjR7nzzjszraFOnTqO+0WLFiUoKIgTJ04A0K9fPx544AF27tzJ3XffTefOnWnevHmOPquI5C+FGxGxRNGiRdOdJnKVgICALO1XpEgRp8c2m42UlBQA2rdvz8GDB1m5ciXr1q3jzjvvpH///rz88ssur1dEXEtzbkSkQPrmm2/SPa5evToA1atX54cffuD8+fOO57ds2YKXlxdVq1alePHiREdHs2HDhlzVUKZMGeLi4njvvfeYMWMG8+fPz9X7iUj+0MiNiFji8uXLJCQkOG3z8fFxTNr9+OOPadSoEf/4xz94//332b59O2+++SYAPXr0YNy4ccTFxTF+/HhOnjzJwIED6dmzJ6GhoQCMHz+eJ554grJly9K+fXvOnj3Lli1bGDhwYJbqGzt2LA0bNqRmzZpcvnyZzz77zBGuRKRgU7gREUusXr2a8PBwp21Vq1bl119/BcyVTB9++CFPPvkk4eHhLFq0iBo1agAQGBjImjVrGDx4MI0bNyYwMJAHHniAadOmOd4rLi6OS5cuMX36dIYNG0bp0qXp2rVrluvz9fVl5MiRHDhwgICAAFq0aMGHH37ogk8uInnNZhiGYXURIiJp2Ww2li5dSufOna0uRUQKIc25EREREbeicCMiIiJuRXNuRKTA0dlyEckNjdyIiIiIW1G4EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiIiIW/k/BPLpH1Zbfp0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = history_dict['binary_accuracy']\n",
    "val_acc = history_dict['val_binary_accuracy']\n",
    "loss = history_dict['loss']\n",
    "val_loss = history_dict['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "# \"bo\" is for \"blue dot\"\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "# b is for \"solid blue line\"\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "af51178e-fe0b-40ca-9260-2190fb52d960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcgUlEQVR4nO3dd1hT598G8DuEvVWQJQKideJWioraSovaUnGideCottaNtmoVR63SWqu4qrV11Um1aP1Vq1WqdY+690RRFBQHCCpKOO8f5000EjCBwEnI/bmuXCZPTk6+J9Dm5pxnyARBEEBERERkQsykLoCIiIiopDEAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiIiMjkMQERERGRyGICIiIjI5DAAEelB79694evrW6jXTpo0CTKZTL8FGZjr169DJpNh2bJlJfq+u3btgkwmw65du1Rt2v6siqtmX19f9O7dW6/7JCLdMQBRqSaTybS6vfoFSVRU+/fvx6RJk/Do0SOpSyGifJhLXQBRcVqxYoXa419//RXbt2/P0169evUivc/PP/+M3NzcQr12/PjxGDNmTJHen7RXlJ+Vtvbv34/Jkyejd+/ecHZ2Vnvu4sWLMDPj355EUmMAolKtR48eao8PHjyI7du352l/3ZMnT2Bra6v1+1hYWBSqPgAwNzeHuTn/UywpRflZ6YOVlZWk728ssrKyYGdnJ3UZVIrxzxAyeS1btkStWrVw9OhRNG/eHLa2tvjqq68AAH/88Qc++OADeHp6wsrKCv7+/pgyZQoUCoXaPl7vV6LsPzJjxgwsWrQI/v7+sLKyQqNGjXDkyBG112rqAySTyTB48GBs3LgRtWrVgpWVFWrWrImtW7fmqX/Xrl1o2LAhrK2t4e/vj59++knrfkV79uxB586dUbFiRVhZWcHb2xsjRozA06dP8xyfvb09kpOTER4eDnt7e7i6umLUqFF5PotHjx6hd+/ecHJygrOzMyIjI7W6FPTff/9BJpNh+fLleZ7btm0bZDIZ/vzzTwDAjRs38Pnnn6Nq1aqwsbFBuXLl0LlzZ1y/fv2N76OpD5C2NZ86dQq9e/dGpUqVYG1tDXd3d/Tt2xf3799XbTNp0iR88cUXAAA/Pz/VZVZlbZr6AF27dg2dO3dG2bJlYWtri7fffhubN29W20bZn+m3337D1KlTUaFCBVhbW6NVq1a4cuXKG49bl8/s0aNHGDFiBHx9fWFlZYUKFSqgV69eSEtLU23z7NkzTJo0CW+99Rasra3h4eGBDh064OrVq2r1vn55WVPfKuXv19WrV9G2bVs4ODige/fuALT/HQWACxcuoEuXLnB1dYWNjQ2qVq2KcePGAQB27twJmUyGDRs25Hnd6tWrIZPJcODAgTd+jlR68M9OIgD3799HmzZt0LVrV/To0QNubm4AgGXLlsHe3h5RUVGwt7fHP//8gwkTJiAjIwPff//9G/e7evVqPH78GJ9++ilkMhmmT5+ODh064Nq1a288E7F3717Ex8fj888/h4ODA+bMmYOOHTsiKSkJ5cqVAwAcP34crVu3hoeHByZPngyFQoGvv/4arq6uWh33unXr8OTJEwwcOBDlypXD4cOHMXfuXNy6dQvr1q1T21ahUCA0NBSBgYGYMWMGduzYgR9++AH+/v4YOHAgAEAQBLRr1w579+7FZ599hurVq2PDhg2IjIx8Yy0NGzZEpUqV8Ntvv+XZPi4uDmXKlEFoaCgA4MiRI9i/fz+6du2KChUq4Pr161iwYAFatmyJc+fO6XT2Tpeat2/fjmvXrqFPnz5wd3fH2bNnsWjRIpw9exYHDx6ETCZDhw4dcOnSJaxZswazZs2Ci4sLAOT7M0lNTUWTJk3w5MkTDB06FOXKlcPy5cvx0UcfYf369Wjfvr3a9t9++y3MzMwwatQopKenY/r06ejevTsOHTpU4HFq+5llZmYiODgY58+fR9++fVG/fn2kpaVh06ZNuHXrFlxcXKBQKPDhhx8iISEBXbt2xbBhw/D48WNs374dZ86cgb+/v9afv1JOTg5CQ0PRrFkzzJgxQ1WPtr+jp06dQnBwMCwsLDBgwAD4+vri6tWr+N///oepU6eiZcuW8Pb2xqpVq/J8pqtWrYK/vz+CgoJ0rpuMmEBkQgYNGiS8/mvfokULAYCwcOHCPNs/efIkT9unn34q2NraCs+ePVO1RUZGCj4+PqrHiYmJAgChXLlywoMHD1Ttf/zxhwBA+N///qdqmzhxYp6aAAiWlpbClStXVG0nT54UAAhz585VtYWFhQm2trZCcnKyqu3y5cuCubl5nn1qoun4YmJiBJlMJty4cUPt+AAIX3/9tdq29erVExo0aKB6vHHjRgGAMH36dFVbTk6OEBwcLAAQli5dWmA9Y8eOFSwsLNQ+s+zsbMHZ2Vno27dvgXUfOHBAACD8+uuvqradO3cKAISdO3eqHcurPytdatb0vmvWrBEACLt371a1ff/99wIAITExMc/2Pj4+QmRkpOrx8OHDBQDCnj17VG2PHz8W/Pz8BF9fX0GhUKgdS/Xq1YXs7GzVtrNnzxYACKdPn87zXq/S9jObMGGCAECIj4/Ps31ubq4gCIKwZMkSAYAwc+bMfLfR9NkLwsv/Nl79XJW/X2PGjNGqbk2/o82bNxccHBzU2l6tRxDE3y8rKyvh0aNHqra7d+8K5ubmwsSJE/O8D5VuvARGBLFfRp8+ffK029jYqO4/fvwYaWlpCA4OxpMnT3DhwoU37jciIgJlypRRPQ4ODgYgXvJ4k5CQELW/pGvXrg1HR0fVaxUKBXbs2IHw8HB4enqqtqtcuTLatGnzxv0D6seXlZWFtLQ0NGnSBIIg4Pjx43m2/+yzz9QeBwcHqx3Lli1bYG5urjojBAByuRxDhgzRqp6IiAi8ePEC8fHxqra///4bjx49QkREhMa6X7x4gfv376Ny5cpwdnbGsWPHtHqvwtT86vs+e/YMaWlpePvttwFA5/d99f0bN26MZs2aqdrs7e0xYMAAXL9+HefOnVPbvk+fPrC0tFQ91vZ3StvP7Pfff0edOnXynCUBoLqs+vvvv8PFxUXjZ1SUKR1e/Rloqju/39F79+5h9+7d6Nu3LypWrJhvPb169UJ2djbWr1+vaouLi0NOTs4b+wVS6cMARATAy8tL7UtF6ezZs2jfvj2cnJzg6OgIV1dX1f8o09PT37jf1/9nrAxDDx8+1Pm1ytcrX3v37l08ffoUlStXzrOdpjZNkpKS0Lt3b5QtW1bVr6dFixYA8h6ftbV1nss4r9YDiP1MPDw8YG9vr7Zd1apVtaqnTp06qFatGuLi4lRtcXFxcHFxwbvvvqtqe/r0KSZMmABvb29YWVnBxcUFrq6uePTokVY/l1fpUvODBw8wbNgwuLm5wcbGBq6urvDz8wOg3e9Dfu+v6b2UIxNv3Lih1l7Y3yltP7OrV6+iVq1aBe7r6tWrqFq1ql4775ubm6NChQp52rX5HVWGvzfVXa1aNTRq1AirVq1Sta1atQpvv/221v/NUOnBPkBEUP8rU+nRo0do0aIFHB0d8fXXX8Pf3x/W1tY4duwYRo8erdVQarlcrrFdEIRifa02FAoF3nvvPTx48ACjR49GtWrVYGdnh+TkZPTu3TvP8eVXj75FRERg6tSpSEtLg4ODAzZt2oRu3bqpfdkOGTIES5cuxfDhwxEUFAQnJyfIZDJ07dq1WIe4d+nSBfv378cXX3yBunXrwt7eHrm5uWjdunWxD61XKuzvRUl/ZvmdCXq907ySlZVVnukBdP0d1UavXr0wbNgw3Lp1C9nZ2Th48CDmzZun837I+DEAEeVj165duH//PuLj49G8eXNVe2JiooRVvVS+fHlYW1trHAGkzaig06dP49KlS1i+fDl69eqlat++fXuha/Lx8UFCQgIyMzPVzqhcvHhR631ERERg8uTJ+P333+Hm5oaMjAx07dpVbZv169cjMjISP/zwg6rt2bNnhZp4UNuaHz58iISEBEyePBkTJkxQtV++fDnPPnW5DOTj46Px81FeYvXx8dF6XwXR9jPz9/fHmTNnCtyXv78/Dh06hBcvXuTbmV95Zur1/b9+Rqsg2v6OVqpUCQDeWDcAdO3aFVFRUVizZg2ePn0KCwsLtcurZDp4CYwoH8q/tF/9y/r58+f48ccfpSpJjVwuR0hICDZu3Ijbt2+r2q9cuYK//vpLq9cD6scnCAJmz55d6Jratm2LnJwcLFiwQNWmUCgwd+5crfdRvXp1BAQEIC4uDnFxcfDw8FALoMraXz/jMXfu3HzPLuijZk2fFwDExsbm2ady/hptAlnbtm1x+PBhtSHYWVlZWLRoEXx9fVGjRg1tD6VA2n5mHTt2xMmTJzUOF1e+vmPHjkhLS9N45kS5jY+PD+RyOXbv3q32vC7//Wj7O+rq6ormzZtjyZIlSEpK0liPkouLC9q0aYOVK1di1apVaN26tWqkHpkWngEiykeTJk1QpkwZREZGYujQoZDJZFixYoXeLkHpw6RJk/D333+jadOmGDhwIBQKBebNm4datWrhxIkTBb62WrVq8Pf3x6hRo5CcnAxHR0f8/vvvWvVPyk9YWBiaNm2KMWPG4Pr166hRowbi4+N17h8TERGBCRMmwNraGv369ctzaeTDDz/EihUr4OTkhBo1auDAgQPYsWOHanqA4qjZ0dERzZs3x/Tp0/HixQt4eXnh77//1nhGsEGDBgCAcePGoWvXrrCwsEBYWJjGif3GjBmDNWvWoE2bNhg6dCjKli2L5cuXIzExEb///rveZo3W9jP74osvsH79enTu3Bl9+/ZFgwYN8ODBA2zatAkLFy5EnTp10KtXL/z666+IiorC4cOHERwcjKysLOzYsQOff/452rVrBycnJ3Tu3Blz586FTCaDv78//vzzT9y9e1frmnX5HZ0zZw6aNWuG+vXrY8CAAfDz88P169exefPmPP8t9OrVC506dQIATJkyRfcPk0qHEh93RiSh/IbB16xZU+P2+/btE95++23BxsZG8PT0FL788kth27ZtbxxarRzq+/333+fZJwC1Ibf5DYMfNGhQnte+PoRaEAQhISFBqFevnmBpaSn4+/sLv/zyizBy5EjB2to6n0/hpXPnzgkhISGCvb294OLiIvTv31813P71Ycp2dnZ5Xq+p9vv37ws9e/YUHB0dBScnJ6Fnz57C8ePHtRoGr3T58mUBgABA2Lt3b57nHz58KPTp00dwcXER7O3thdDQUOHChQt5Ph9thsHrUvOtW7eE9u3bC87OzoKTk5PQuXNn4fbt23l+poIgCFOmTBG8vLwEMzMztSHxmn6GV69eFTp16iQ4OzsL1tbWQuPGjYU///xTbRvlsaxbt06tXdOwck20/cyUn8fgwYMFLy8vwdLSUqhQoYIQGRkppKWlqbZ58uSJMG7cOMHPz0+wsLAQ3N3dhU6dOglXr15VbXPv3j2hY8eOgq2trVCmTBnh008/Fc6cOaP175cgaP87KgiCcObMGdXPx9raWqhataoQHR2dZ5/Z2dlCmTJlBCcnJ+Hp06cFfm5UeskEwYD+nCUivQgPD8fZs2c19k8hMnU5OTnw9PREWFgYFi9eLHU5JBH2ASIycq8vCXD58mVs2bIFLVu2lKYgIgO3ceNG3Lt3T61jNZkengEiMnIeHh6q9alu3LiBBQsWIDs7G8ePH0eVKlWkLo/IYBw6dAinTp3ClClT4OLiUujJK6l0YCdoIiPXunVrrFmzBikpKbCyskJQUBCmTZvG8EP0mgULFmDlypWoW7eu2mKsZJp4BoiIiIhMDvsAERERkclhACIiIiKTwz5AGuTm5uL27dtwcHAo0srGREREVHIEQcDjx4/h6en5xklEGYA0uH37Nry9vaUug4iIiArh5s2bqFChQoHbMABp4ODgAED8AB0dHSWuhoiIiLSRkZEBb29v1fd4QRiANFBe9nJ0dGQAIiIiMjLadF9hJ2giIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOZwJmoiIiEqEQgHs2QPcuQN4eADBwYBcLk0tDEBERERU7OLjgWHDgFu3XrZVqADMng106FDy9fASGBERERWr+HigUyf18AMAyclie3x8ydfEAERERETFRqEQz/wIQt7nlG3Dh4vblSQGICIiIio2e/bkPfPzKkEAbt4UtytJDEBERERUbO7c0e92+sIARERERMXGw0O/2+kLR4EREREZOEMaPq6r4GBxtFdysuZ+QDKZ+HxwcMnWxTNAREREBiw+HvD1Bd55B/j4Y/FfX19pRk4VhlwuDnUHxLDzKuXj2NiSD3QMQERERAbKEIePF0aHDsD69YCXl3p7hQpiuxTzAMkEQdMJKdOWkZEBJycnpKenw9HRUepyiIjIBCkU4pme/EZQKS8dJSYaz+Ww4r6Up8v3N/sAERERGSBdho+3bFliZRWJXG44tfISGBERkQEy1OHjpQUDEBERkQEy1OHjpYXkAWj+/Pnw9fWFtbU1AgMDcfjw4Xy3ffHiBb7++mv4+/vD2toaderUwdatW4u0TyIiIkOkHD7++sgpJZkM8PYu+eHjpYWkASguLg5RUVGYOHEijh07hjp16iA0NBR3797VuP348ePx008/Ye7cuTh37hw+++wztG/fHsePHy/0PomIqPRSKIBdu4A1a8R/S3q9qaIw1OHjpYYgocaNGwuDBg1SPVYoFIKnp6cQExOjcXsPDw9h3rx5am0dOnQQunfvXuh9apKeni4AENLT07V+DRERGZbffxeEChUEQewuLN4qVBDbjYmm4/D2Nr7jKAm6fH9Ldgbo+fPnOHr0KEJCQlRtZmZmCAkJwYEDBzS+Jjs7G9bW1mptNjY22Lt3b6H3SUREpU9pmT8HEOfIuX4d2LkTWL1a/DcxUZq5c0oTyYbBp6WlQaFQwM3NTa3dzc0NFy5c0Pia0NBQzJw5E82bN4e/vz8SEhIQHx8Pxf+f0yzMPgExWGVnZ6seZ2RkFPawiIhIYgoFMGyY5mUXBEG8fDR8ONCunfFcPjKk4eOlheSdoHUxe/ZsVKlSBdWqVYOlpSUGDx6MPn36wMysaIcRExMDJycn1c3b21tPFRMRUUnTZf4cMl2SBSAXFxfI5XKkpqaqtaempsLd3V3ja1xdXbFx40ZkZWXhxo0buHDhAuzt7VGpUqVC7xMAxo4di/T0dNXt5s2bRTw6IiKSCufPIW1IFoAsLS3RoEEDJCQkqNpyc3ORkJCAoKCgAl9rbW0NLy8v5OTk4Pfff0e7du2KtE8rKys4Ojqq3YiIyDhx/hzShqRLYURFRSEyMhINGzZE48aNERsbi6ysLPTp0wcA0KtXL3h5eSEmJgYAcOjQISQnJ6Nu3bpITk7GpEmTkJubiy+//FLrfRIRUemmnD8nOVlzPyDlGlqcP8e0SRqAIiIicO/ePUyYMAEpKSmoW7cutm7dqurEnJSUpNa/59mzZxg/fjyuXbsGe3t7tG3bFitWrICzs7PW+yQiojcr7kUri5Ny/pxOncSw82oI4vw5pMTV4DXgavBEZMri48VRVK92JK5QQQwVxjT0WtNxeHuL4ceYjoO0p8v3NwOQBgxARGSqlPPnvP7NoDxzsn69cYUHYz6TRbpjACoiBiAiMkUKBeDrm/8QcmXfmcREhggyTLp8fxvVPEBERFR8OH8OmRIGICIiAsD5c8i0MAAREREAzp9DpoUBiIiIALycP0fZ4fl1Mpk4iorz51BpwABEREQAXs6fA+QNQZw/h0obBiAiIlLp0EEc6u7lpd5eoYLxDYEnKoikM0ETEZHh6dABaNeO8+dQ6cYAREREecjlQMuWUldBVHx4CYyIiIhMDs8AERHpEZdeIDIODEBERHpSWhYRJTIFvARGRKQHykVEX19KIjlZbI+Pl6YuItKMAYiIqIgUCvHMj6alpZVtw4eL2xGRYWAAIiIqIi4iSmR8GICIiIqIi4gSGR8GICKiIuIiokTGhwGIiKiIuIgokfFhACIiKiIuIkpkfBiAiIj0gIuIEhkXToRIRKQnXESU9CUnB0hLA+7efXm7dw94+BAwMwPMzcWbXK7+75vaivr8621mZvlf+jV0DEBERHrERURJk9xcMby8GmZeDTevtz14IHXF2itsEOvdGxg8WMK6pXtrIiIi4yQIwOPH2oWZu3fFszm6ToRpZga4uADly4s3V1egTBnxvRUK8SxRTs7L+5ratLmvzbaaJvlUUm6Tna3b8YWG6ra9vjEAEZFB4CKiJLWnT7ULM8rHun7hA2KAUYYZZbB5NeC8+rhMGcP5byA3V/xvVJ8hq1IlaY+JAYiIJMdFRKm4PH0K3LghzsRdUJi5exfIzNR9//b22oUZV1fxbI6lpf6PsSSYmYk3CwupK9EfBiAikpRyEdHXT7ErFxHlCCoqyIsXQFIScP06kJgo3l69n5Ki2/4sLbULM8p/bW2L46ioJMgEoaAre6YpIyMDTk5OSE9Ph6Ojo9TlEJVaCgXg65v/OloymXgmKDHRcC4FUMlSKMQwrAw1rwedW7fEyzMFcXAAKlYE3N3zDzPKm4OD8Y5qIt2+v3kGiIgko8siohxZVToJApCaqvnszfXr4tmdFy8K3oe1NeDnJ4bpV/9V3i9blqGG8mIAIiLJcBHR0k8QxCHdmsKN8t9nzwreh4WFeAbn9XCjfOzmxoBDumMAIiLJcBHR0iEjI/8+ONevi8PFC2JmJl7qfP3MjfK+pycvgZL+MQARkWSUi4gmJ2ueZ0TZB4iLiErryRNxJFV+AUebSfvc3TWHG19fcaFYYx0dRcaLAYiIJKNcRLRTJzHsvBqCSuMiogqF2J/l1VtOTt42Kdtff+7RI7GPzpuUK6c53Pj5AT4+gI1NMX+4RDpiACIiSSkXEdU0D1BsrOEPgX/8GLh4EbhwQf2Wmpo3VBjzmFsHh7x9b1697+AgdYVEumEAIiLJGfoiooIghrPXQ87Fi+Llu6IwNxc7+Spvrz8uqfb8nrO3FwNOmTLsaEylCwMQERkEQ1hE9Nkz4PJlzUEnKyv/17m7A9Wqqd+8vMR+LQUFEXNzhgoiqTAAEZFJEQRx+YPXQ86FC2KH3vwuU5mbA1WqiOGmatWXQadqVcDZuSSPgIj0gQGIiEqlFy+Aa9c09895+DD/1zk7A9Wr5z2j4+dXutZBIjJ1DEBEZNQePdIccq5cETseayKTiYHm9bM51aqJSyPwshRR6ccAREQGLzdXXBJBU9+cgha7tLXNeyanWjWgcmUOyyYydQxARGQwnjwBLl3KG3QuXQKePs3/dV5eec/kKDsim5mVXP1EZDwYgIioxAmC2OH41Cn12+XL+XdCtrR82Qn51dtbbwFvWPSZiCgPBiAiI6dQGO78OYC4TtTp0+pB5/Tp/NeHKltWcydkX19xJBYRkT7wfydERiw+XvMMyrNnl/wMygqF2PH49bM6169r3t7SEqhRA6hdW/3m5laiZRORiWIAIjJS8fHiGlqvXzJKThbb168vvhCUlpb3rM6ZM+JEgppUqJA36Lz1FoeVE5F0ZIJgzKvTFI+MjAw4OTkhPT0djuxcQAZIoRAvCb165udVylXUExOLdjns+XNxpNXrZ3Vu39a8va0tUKuWetAJCBAvaxERFTddvr95BojICO3Zk3/4AcSzQjdvittps7yEIIjDyV8POufPixMKalKpUt6zOpUqGVb/IyKi/DAAERmhO3cKv93Tp8DZs+odkk+dEi9raeLk9PJMjjLo1KrF1b+JyLgxABEZIQ8P7baTyYBNm/IONc/NzbutmZk4l87rZ3W8vTkzMhGVPpJPETZ//nz4+vrC2toagYGBOHz4cIHbx8bGomrVqrCxsYG3tzdGjBiBZ6/0vJw0aRJkMpnarVq1asV9GEQlKjhY7ONTUDCRyYBu3YB27YDoaGDdOrE/T24u4OICtGoFjBgBLF0KHD0KZGYC584Ba9cCX30FfPghULEiww8RlU6SngGKi4tDVFQUFi5ciMDAQMTGxiI0NBQXL15E+fLl82y/evVqjBkzBkuWLEGTJk1w6dIl9O7dGzKZDDNnzlRtV7NmTezYsUP12JyTh1ApI5cDs2YBnTvnv40giKOs8htqzmBDRKZM0mQwc+ZM9O/fH3369AEALFy4EJs3b8aSJUswZsyYPNvv378fTZs2xccffwwA8PX1Rbdu3XDo0CG17czNzeHu7l78B0AkgeRkYNkyYMkSzc87OAD9+om3qlU51JyISBPJAtDz589x9OhRjB07VtVmZmaGkJAQHDhwQONrmjRpgpUrV+Lw4cNo3Lgxrl27hi1btqBnz55q212+fBmenp6wtrZGUFAQYmJiULFixXxryc7ORnZ2tupxRkZGEY+OSL+ePwf+/BNYvBjYuvVlHx4HB6BrV6B+fXE5CE9Pw5sJmojIEEkWgNLS0qBQKOD22rSvbm5uuHDhgsbXfPzxx0hLS0OzZs0gCAJycnLw2Wef4auvvlJtExgYiGXLlqFq1aq4c+cOJk+ejODgYJw5cwYO+QxbiYmJweTJk/V3cER6cuGCGHqWLwfu3XvZHhwsnuHp1Amws5OuPiIiYyV5J2hd7Nq1C9OmTcOPP/6IY8eOIT4+Hps3b8aUKVNU27Rp0wadO3dG7dq1ERoaii1btuDRo0f47bff8t3v2LFjkZ6errrdvHmzJA6HSKPMTPHyVtOm4ppYM2aI4cfNDRg9WuzIvHs3EBnJ8ENEVFiSnQFycXGBXC5HamqqWntqamq+/Xeio6PRs2dPfPLJJwCAgIAAZGVlYcCAARg3bhzMzPLmOWdnZ7z11lu4cuVKvrVYWVnBysqqCEdDVDSCABw6JJ7tWbtWDEGAeCmrbVvxbE/btuzPQ0SkL5KdAbK0tESDBg2QkJCgasvNzUVCQgKCgoI0vubJkyd5Qo78/zs75LeiR2ZmJq5evQoPbSdOISpB9+4BM2eKEwsGBQG//CKGn8qVgZgYIClJnMenXTuGHyIifZJ0FFhUVBQiIyPRsGFDNG7cGLGxscjKylKNCuvVqxe8vLwQExMDAAgLC8PMmTNRr149BAYG4sqVK4iOjkZYWJgqCI0aNQphYWHw8fHB7du3MXHiRMjlcnTr1k2y4yR6lUIBbN8unu3544+XS03Y2Ih9evr1A5o35zB1IqLiJGkAioiIwL179zBhwgSkpKSgbt262Lp1q6pjdFJSktoZn/Hjx0Mmk2H8+PFITk6Gq6srwsLCMHXqVNU2t27dQrdu3XD//n24urqiWbNmOHjwIFxdXUv8+Ihedf262Ldn2TJxnS6lBg2ATz4RJy10cpKqOiIi08LV4DXgavCkL8+eARs3imd7EhLEvj4AUKYM0KOHeLanTh1JSyQiKjW4GjyRxE6eFEPPypXAw4cv21u1Es/2hIcD1taSlUdEZPIYgIj0JD0dWLNGDD7//feyvUIFoE8f8ebnJ119RET0EgMQUREIgjgnz+LFwPr1wNOnYruFhThyq18/4L33ODMzEZGhYQAiKoQ7d8TZmZcsAS5fftleo4YYenr2BNjvnojIcDEAEWkpJwfYskWcq2fLFnE4OwDY2wMREWLfnsBADl8nIjIGDEBEb3DpknimZ/lyICXlZXuTJuLZni5dxBBERETGgwGISIOsLLFPz+LFwJ49L9tdXcU1uPr2FdfpIiIi48QARPT/BEEcvbV4sTiaKyNDbDczA1q3Fs/2fPghYGkpbZ1ERFR0DEBkshQK8ezOpUvAqVPiaK7Tp18+7+cnhp7ISHEoOxERlR4MQGSS4uOBIUOA27fV2y0sgM6dxeDTsqV49oeIiEofBiAyOfHxQMeOmp978UJ87t13S7YmIiIqWfz7lkyKQgEMGpT/8zIZMHz4yyHuRERUOjEAkUlZvVp9KPvrBEFcqf3VkV9ERFT6MACRyTh1Chg8WLtt79wp3lqIiEhaDEBkEg4eBFq0eDm0/U08PIq3HiIikhYDEJV6O3YAISHAo0dAUBDg6Zn/chUyGeDtDQQHl2iJRERUwhiAqFT74w/ggw/EmZ3ffx/Yvh2YO1d87vUQpHwcG8vV24mISjsGICq1Vq4Uh7Q/fw506ABs2gTY2Yn3168HvLzUt69QQWzv0EGaeomIqORwHiAqlX788eVw98hIcQV381d+2zt0ANq1E0d73bkj9vkJDuaZHyIiU8EARKVOTAzw1Vfi/SFDxEtammZ0lsvF2Z6JiMj08BIYlRqCAIwZ8zL8REcDs2dzOQsiIsqLZ4CoVMjNFS95LVwoPp4xAxg5UtqaiIjIcDEAkdF78QLo3Vuc5VkmA376CejfX+qqiIjIkDEAkVF79gyIiBBHeJmbAytWAF27Sl0VEREZOgYgMlqPH4sjuXbuBKytxSHsH3wgdVVERGQMGIDIKD14ALRtCxw6BNjbA3/+KS51QUREpA0GIDI6KSnirM6nTwNlywJbtwKNGkldFRERGRMGIDIqN26I63pduSJOXvj330CtWlJXRURExoYBiIzGhQvAe+8Bt24Bvr7iIqf+/lJXRURExohTxJFROH4caN5cDD/VqwN79zL8EBFR4TEAkcHbtw945x3g3j2gQQNg9+68C5kSERHpggGIDNrff4sdntPTxcVKExIAFxepqyIiImPHAEQG6/ffgQ8/BJ48AVq3Fkd7OTlJXRUREZUGDEBkkJYvB7p0EZe56NwZ+OMPwNZW6qqIiKi0YAAigzN3rri2V24u0K8fsGYNYGkpdVVERFSaMACRwRAE4JtvgKFDxccjRgA//wzI5dLWRUREpQ8DEBkEQQC++AKIjhYfT5oE/PCDuLo7ERGRvnEiRJKcQgEMHCie7QGAWbOA4cMlLYmIiEo5BiCS1PPnQK9eQFwcYGYmhqC+faWuioiISjsGIJLM06dAp07Ali2AhQWwapU44ouIiKi4MQCRJDIygI8+Av79F7CxAeLjxbl+iIiISgIDEJW4+/fFsPPff4CDA7B5szjLMxERUUlhAKISdfu2uKL7uXPikhZbt4rrexEREZUkBiAqMdeuASEhQGIi4OkJ7NghruxORERU0jgPEJWIc+fEy1yJiUClSsDevQw/REQkHQYgKnZHjwLNm4uXv2rWBPbsAfz8pK6KiIhMGQMQFavdu4F33hE7PjdqJI768vSUuioiIjJ1DEBUbLZsAUJDgcePgRYtgIQEoFw5qasiIiIygAA0f/58+Pr6wtraGoGBgTh8+HCB28fGxqJq1aqwsbGBt7c3RowYgWfPnhVpn6R/v/0GtGsHPHsGfPAB8Ndf4pB3IiIiQyBpAIqLi0NUVBQmTpyIY8eOoU6dOggNDcXdu3c1br969WqMGTMGEydOxPnz57F48WLExcXhq6++KvQ+Sf8WLwa6dQNycoCuXYENG8TJDomIiAyFTBAEQao3DwwMRKNGjTBv3jwAQG5uLry9vTFkyBCMGTMmz/aDBw/G+fPnkZCQoGobOXIkDh06hL179xZqn5pkZGTAyckJ6enpcHR0LOphmpRZs4CoKPF+//7AggWAXC5tTUREZBp0+f6W7AzQ8+fPcfToUYSEhLwsxswMISEhOHDggMbXNGnSBEePHlVd0rp27Rq2bNmCtm3bFnqfAJCdnY2MjAy1G+lGEICJE1+Gn1GjgJ9+YvghIiLDJNlEiGlpaVAoFHBzc1Nrd3Nzw4ULFzS+5uOPP0ZaWhqaNWsGQRCQk5ODzz77THUJrDD7BICYmBhMnjy5iEdkunJzxeAze7b4+JtvgK++AmQyaesiIiLKj+SdoHWxa9cuTJs2DT/++COOHTuG+Ph4bN68GVOmTCnSfseOHYv09HTV7ebNm3qquPRTKIBPPnkZfubMAcaNY/ghIiLDJtkZIBcXF8jlcqSmpqq1p6amwt3dXeNroqOj0bNnT3zyyScAgICAAGRlZWHAgAEYN25cofYJAFZWVrCysiriEZme7GygRw9g/XrAzAxYsgSIjJS6KiIiojeT7AyQpaUlGjRooNahOTc3FwkJCQgKCtL4midPnsDMTL1k+f93MhEEoVD7pMJ58kQc5r5+PWBpCaxbx/BDRETGQ9LFUKOiohAZGYmGDRuicePGiI2NRVZWFvr06QMA6NWrF7y8vBATEwMACAsLw8yZM1GvXj0EBgbiypUriI6ORlhYmCoIvWmfVHRZWUDr1uJ6Xra2wMaN4grvRERExkLSABQREYF79+5hwoQJSElJQd26dbF161ZVJ+akpCS1Mz7jx4+HTCbD+PHjkZycDFdXV4SFhWHq1Kla75OKbtYsMfw4OQGbNwNNm0pdERERkW4knQfIUHEeoPw9ewb4+AB37wIrVwLdu0tdERERkahY5wHy9fXF119/jaSkpEIXSMZr5Uox/Hh7A126SF0NERFR4egcgIYPH474+HhUqlQJ7733HtauXYvs7OziqI0MTG4u8MMP4v3hwwELC0nLISIiKrRCBaATJ07g8OHDqF69OoYMGQIPDw8MHjwYx44dK44ayUBs2QJcuAA4Oopz/xARERmrQg+Dr1+/PubMmYPbt29j4sSJ+OWXX9CoUSPUrVsXS5YsAbsWlT4zZoj/fvqpGIKIiIiMVaFHgb148QIbNmzA0qVLsX37drz99tvo168fbt26ha+++go7duzA6tWr9VkrSejIEeDffwFzc2DoUKmrISIiKhqdA9CxY8ewdOlSrFmzBmZmZujVqxdmzZqFatWqqbZp3749GjVqpNdCSVrKvj/dugEVKkhbCxERUVHpHIAaNWqE9957DwsWLEB4eDgsNPSE9fPzQ9euXfVSIEnv+nVxpmcAGDlS0lKIiIj0QucAdO3aNfj4+BS4jZ2dHZYuXVroosiwxMaKI8Deew+oU0fqaoiIiIpO507Qd+/exaFDh/K0Hzp0CP/9959eiiLD8fAh8Msv4v1Ro6SthYiISF90DkCDBg3CzZs387QnJydj0KBBeimKDMdPP4lrfwUEcL0vIiIqPXQOQOfOnUP9+vXztNerVw/nzp3TS1FkGLKzgTlzxPujRgEymbT1EBER6YvOAcjKygqpqal52u/cuQNzc0nXViU9W7MGuHMH8PQE2KediIhKE50D0Pvvv4+xY8ciPT1d1fbo0SN89dVXeI/XSEoNQXg58eGwYYClpbT1EBER6ZPOp2xmzJiB5s2bw8fHB/Xq1QMAnDhxAm5ublixYoXeCyRpbNsGnD0L2NsDAwZIXQ0REZF+6RyAvLy8cOrUKaxatQonT56EjY0N+vTpg27dummcE4iMk/LsT//+gLOzpKUQERHpnUzgol15ZGRkwMnJCenp6XA0wUWvjh8H6tcH5HLg6lXgDdM+ERERGQRdvr8L3Wv53LlzSEpKwvPnz9XaP/roo8LukgyEctmLLl00hx+FAtizR+wg7eEBBAeLYYmIiMhYFGom6Pbt2+P06dOQyWSqVd9l/z9GWqFQ6LdCKlE3bwJr14r3NS17ER8vdoq+detlW4UKwOzZQIcOJVMjERFRUek8CmzYsGHw8/PD3bt3YWtri7Nnz2L37t1o2LAhdu3aVQwlUkmaPVs8w/POO0CDBurPxccDnTqphx8ASE4W2+PjS65OIiKiotA5AB04cABff/01XFxcYGZmBjMzMzRr1gwxMTEYOnRocdRIJSQ9HVi0SLz/+rIXCoV45kdTjzFl2/Dh4nZERESGTucApFAo4ODgAABwcXHB7du3AQA+Pj64ePGifqujEvXzz8Djx0CNGkDr1urP7dmT98zPqwRBvHy2Z0/x1khERKQPOvcBqlWrFk6ePAk/Pz8EBgZi+vTpsLS0xKJFi1CpUqXiqJFKwPPn4qrvgNj3x+y1aHznjnb70XY7IiIiKekcgMaPH4+srCwAwNdff40PP/wQwcHBKFeuHOLi4vReIJWM334T+/K4uQHdu+d93sNDu/1oux0REZGU9DIP0IMHD1CmTBnVSDBjZ2rzAAkCUK8ecPIkMHUq8NVXebdRKABfXzEkafqNkcnE0WCJiRwST0RE0tDl+1unPkAvXryAubk5zpw5o9ZetmzZUhN+TFFCghh+bG2Bzz7TvI1cLo4QA/KuCq98HBvL8ENERMZBpwBkYWGBihUrcq6fUka57EW/fkDZsvlv16EDsH494OWl3l6hgtjOeYCIiMhY6HwJbPHixYiPj8eKFStQtqBvSyNmSpfATp0C6tQROz1fvgxo04+dM0ETEZEhKtalMObNm4crV67A09MTPj4+sLOzU3v+2LFjuu6SJDRzpvhvx47ahR9ADDstWxZbSURERMVO5wAUHh5eDGWQFJKTgdWrxfualr0gIiIqrXQOQBMnTiyOOkgCc+cCL16Il7ACA6WuhoiIqOToPBM0lQ6PHwMLF4r3X1/2goiIqLTT+QyQmZlZgUPeOULMOCxeLK799dZbwIcfSl0NERFRydI5AG3YsEHt8YsXL3D8+HEsX74ckydP1lthVHxycoBZs8T7mpa9ICIiKu30MhM0AKxevRpxcXH4448/9LE7SZX2YfBr1wLdugGursCNG4CNjdQVERERFV2xzQRdkLfffhsJCQn62h0VE0EAvv9evD94MMMPERGZJr0EoKdPn2LOnDnwen2KYDI4//4LHDsGWFsDn38udTVERETS0LkP0OuLngqCgMePH8PW1hYrV67Ua3Gkf8plL/r0AVxcpK2FiIhIKjoHoFmzZqkFIDMzM7i6uiIwMBBlypTRa3GkX+fOAZs3i4uXjhghdTVERETS0TkA9e7duxjKoJKgXPYiPByoUkXSUoiIiCSlcx+gpUuXYt26dXna161bh+XLl+ulKNK/lBRgxQrxPic+JCIiU6dzAIqJiYGLhs4j5cuXx7Rp0/RSFOnfvHnA8+dAUBDQpInU1RAREUlL5wCUlJQEPz+/PO0+Pj5ISkrSS1GkX1lZwI8/ivd59oeIiKgQAah8+fI4depUnvaTJ0+iXLlyeimK9GvpUuDhQ8DfH2jXTupqiIiIpKdzAOrWrRuGDh2KnTt3QqFQQKFQ4J9//sGwYcPQtWvX4qiRikCheNn5OSoKkMulrYeIiMgQ6DwKbMqUKbh+/TpatWoFc3Px5bm5uejVqxf7ABmgDRuAxESgXDmAA/iIiIhEOgcgS0tLxMXF4ZtvvsGJEydgY2ODgIAA+Pj4FEd9VASvLnvx+eeAra209RARERkKnQOQUpUqVVCFk8kYtH37gMOHASsrYNAgqashIiIyHDr3AerYsSO+++67PO3Tp09H586d9VIU6Ydy2YtevQA3N2lrISIiMiQ6B6Ddu3ejbdu2edrbtGmD3bt3F6qI+fPnw9fXF9bW1ggMDMThw4fz3bZly5aQyWR5bh988IFqm969e+d5vnXr1oWqzVhdvAhs2iTej4qSthYiIiJDo/MlsMzMTFhaWuZpt7CwQEZGhs4FxMXFISoqCgsXLkRgYCBiY2MRGhqKixcvonz58nm2j4+Px/Pnz1WP79+/jzp16uQ5+9S6dWssXbpU9djKykrn2ozZrFliH6CwMKBaNamrISIiMiw6nwEKCAhAXFxcnva1a9eiRo0aOhcwc+ZM9O/fH3369EGNGjWwcOFC2NraYsmSJRq3L1u2LNzd3VW37du3w9bWNk8AsrKyUtvOlBZqvXsXUK5KwokPiYiI8tL5DFB0dDQ6dOiAq1ev4t133wUAJCQkYPXq1Vi/fr1O+3r+/DmOHj2KsWPHqtrMzMwQEhKCAwcOaLWPxYsXo2vXrrCzs1Nr37VrF8qXL48yZcrg3XffxTfffJPvRI3Z2dnIzs5WPS7MmSxD8uOPwLNnQKNGQHCw1NUQEREZHp3PAIWFhWHjxo24cuUKPv/8c4wcORLJycn4559/ULlyZZ32lZaWBoVCAbfXeui6ubkhJSXlja8/fPgwzpw5g08++UStvXXr1vj111+RkJCA7777Dv/++y/atGkDhUKhcT8xMTFwcnJS3by9vXU6DkPy5Akwf754f9QoQCaTth4iIiJDVKhh8B988IGq03FGRgbWrFmDUaNG4ejRo/mGjOKwePFiBAQEoHHjxmrtr85IHRAQgNq1a8Pf3x+7du1Cq1at8uxn7NixiHqlp3BGRobRhqBffwXS0gBfX6BDB6mrISIiMkw6nwFS2r17NyIjI+Hp6YkffvgB7777Lg4ePKjTPlxcXCCXy5GamqrWnpqaCnd39wJfm5WVhbVr16Jfv35vfJ9KlSrBxcUFV65c0fi8lZUVHB0d1W7G6NVlL0aMAMwLPcsTERFR6aZTAEpJScG3336LKlWqoHPnznB0dER2djY2btyIb7/9Fo0aNdLpzS0tLdGgQQMkJCSo2nJzc5GQkICgoKACX7tu3TpkZ2ejR48eb3yfW7du4f79+/Dw8NCpPmPzv/8Bly8DZcoAfftKXQ0REZHh0joAhYWFoWrVqjh16hRiY2Nx+/ZtzJ07t8gFREVF4eeff8by5ctx/vx5DBw4EFlZWejTpw8AoFevXmqdpJUWL16M8PDwPB2bMzMz8cUXX+DgwYO4fv06EhIS0K5dO1SuXBmhoaFFrteQKSc+HDgQsLeXthYiIiJDpvVFkr/++gtDhw7FwIED9boERkREBO7du4cJEyYgJSUFdevWxdatW1Udo5OSkmBmpp7TLl68iL179+Lvv//Osz+5XI5Tp05h+fLlePToETw9PfH+++9jypQppXouoAMHxKUvLC2BwYOlroaIiMiwyQRBELTZ8ODBg1i8eDHi4uJQvXp19OzZE127doWHhwdOnjxZqDmADFVGRgacnJyQnp5uNP2BOnUCfv9dvPS1eLHU1RAREZU8Xb6/tb4E9vbbb+Pnn3/GnTt38Omnn2Lt2rXw9PREbm4utm/fjsePHxe5cCqcq1eB+HjxPpe9ICIiejOdR4HZ2dmhb9++2Lt3L06fPo2RI0fi22+/Rfny5fHRRx8VR430BsplL9q2BWrWlLoaIiIiw1foYfAAULVqVUyfPh23bt3CmjVr9FUT6eD+fUC5agiXvSAiItJOkQKQklwuR3h4ODYplx+nErNgAfD0KVC/PtCypdTVEBERGQe9BCCSxrNngHImAi57QUREpD0GICO2cqW48nvFiuIoMCIiItIOA5CRys0FfvhBvD98OGBhIWk5RERERoUByEht2QJcuAA4OQGffCJ1NURERMaFAchIKZe9+PRTwMFB2lqIiIiMDQOQETpyBPj3X3G196FDpa6GiIjI+DAAGSFl35+PPwa8vKSthYiIyBgxABmZ69eBdevE+yNHSloKERGR0WIAMjKxseIIsPffB2rXlroaIiIi48QAZEQePgR++UW8z2UviIiICo8ByIj89BOQlSWe+QkJkboaIiIi48UAZCSys4E5c8T7XPaCiIioaBiAjMSaNcCdO+Kor4gIqashIiIybgxARkAQXk58OGwYYGkpbT1ERETGjgHICGzbBpw9K874PGCA1NUQEREZPwYgI6A8+9O/v7j2FxERERUNA5CBO34cSEgA5HLx8hcREREVHQOQgVMuexERAVSsKG0tREREpQUDkAG7eRNYu1a8z2UviIiI9IcByIDNng0oFMC77wL160tdDRERUenBAGSg0tOBRYvE+1z2goiISL8YgAzUzz8Djx8DNWoArVtLXQ0REVHpwgBkgJ4/F1d9B7jsBRERUXFgADJAv/0GJCcD7u7Axx9LXQ0REVHpwwBkYF5d9mLoUMDKStp6iIiISiMGIAOTkACcPAnY2QGffip1NURERKUTA5CBUZ796dcPKFtW2lqIiIhKKwYgA3LqlLjwqZkZMHy41NUQERGVXgxABkS57EWnToCfn7S1EBERlWYMQAbi1i1g9WrxPic+JCIiKl4MQAZi7lwgJwdo3hxo1EjqaoiIiEo3BiADkJEBLFwo3ufZHyIiouLHAGQAFi8WQ1DVqsAHH0hdDRERUenHACSxFy9eLnsxcqQ4AoyIiIiKF79uJbZ+PZCUBJQvD/TsKXU1REREpoEBSEKvLnsxeDBgbS1tPURERKaCAUhCu3YBx44BNjbAwIFSV0NERGQ6GIAkpDz706cP4OIibS1ERESmhAFIImfPAlu2ADIZMGKE1NUQERGZFgYgicycKf7bvj1QubK0tRAREZkaBiAJ3LkDrFwp3ufEh0RERCWPAUgC8+YBz58DTZoAQUFSV0NERGR6GIBKWGYmsGCBeJ9nf4iIiKTBAFTCli4FHj4U+/189JHU1RAREZkmgwhA8+fPh6+vL6ytrREYGIjDhw/nu23Lli0hk8ny3D54ZREtQRAwYcIEeHh4wMbGBiEhIbh8+XJJHEqBcnKAWbPE+1FRgFwubT1ERESmSvIAFBcXh6ioKEycOBHHjh1DnTp1EBoairt372rcPj4+Hnfu3FHdzpw5A7lcjs6dO6u2mT59OubMmYOFCxfi0KFDsLOzQ2hoKJ49e1ZSh6XRhg1AYiJQrhwQGSlpKURERCZN8gA0c+ZM9O/fH3369EGNGjWwcOFC2NraYsmSJRq3L1u2LNzd3VW37du3w9bWVhWABEFAbGwsxo8fj3bt2qF27dr49ddfcfv2bWzcuLEEjyyv48fFeX8GDQJsbSUthYiIyKRJGoCeP3+Oo0ePIiQkRNVmZmaGkJAQHDhwQKt9LF68GF27doWdnR0AIDExESkpKWr7dHJyQmBgYL77zM7ORkZGhtqtOEybBpw/DwwdWiy7JyIiIi1JGoDS0tKgUCjg5uam1u7m5oaUlJQ3vv7w4cM4c+YMPvnkE1Wb8nW67DMmJgZOTk6qm7e3t66HorWqVcVLYERERCQdyS+BFcXixYsREBCAxo0bF2k/Y8eORXp6uup28+ZNPVVIREREhkjSAOTi4gK5XI7U1FS19tTUVLi7uxf42qysLKxduxb9+vVTa1e+Tpd9WllZwdHRUe1GREREpZekAcjS0hINGjRAQkKCqi03NxcJCQkIesMUyevWrUN2djZ69Oih1u7n5wd3d3e1fWZkZODQoUNv3CcRERGZBnOpC4iKikJkZCQaNmyIxo0bIzY2FllZWejTpw8AoFevXvDy8kJMTIza6xYvXozw8HCUe61DjUwmw/Dhw/HNN9+gSpUq8PPzQ3R0NDw9PREeHl5Sh0VEREQGTPIAFBERgXv37mHChAlISUlB3bp1sXXrVlUn5qSkJJiZqZ+ounjxIvbu3Yu///5b4z6//PJLZGVlYcCAAXj06BGaNWuGrVu3wtrautiPh4iIiAyfTBAEQeoiDE1GRgacnJyQnp7O/kBERERGQpfvb6MeBUZERERUGAxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHLMpS6AiIhMi0KhwIsXL6Qug4yQhYUF5HK5XvbFAERERCVCEASkpKTg0aNHUpdCRszZ2Rnu7u6QyWRF2g8DEBERlQhl+ClfvjxsbW2L/AVGpkUQBDx58gR3794FAHh4eBRpfwxARERU7BQKhSr8lCtXTupyyEjZ2NgAAO7evYvy5csX6XIYO0ETEVGxU/b5sbW1lbgSMnbK36Gi9iNjACIiohLDy15UVPr6HZI8AM2fPx++vr6wtrZGYGAgDh8+XOD2jx49wqBBg+Dh4QErKyu89dZb2LJli+r5SZMmQSaTqd2qVatW3IdBRESkFV9fX8TGxmq9/a5duyCTydh5XM8k7QMUFxeHqKgoLFy4EIGBgYiNjUVoaCguXryI8uXL59n++fPneO+991C+fHmsX78eXl5euHHjBpydndW2q1mzJnbs2KF6bG7Ork5ERKWBQgHs2QPcuQN4eADBwYCeRkXn8aYzDRMnTsSkSZN03u+RI0dgZ2en9fZNmjTBnTt34OTkpPN7Uf4kTQYzZ85E//790adPHwDAwoULsXnzZixZsgRjxozJs/2SJUvw4MED7N+/HxYWFgDEJP06c3NzuLu7F2vtRERUsuLjgWHDgFu3XrZVqADMng106KD/97tz547qflxcHCZMmICLFy+q2uzt7VX3BUGAQqHQ6g9uV1dXneqwtLTkd1oxkOwS2PPnz3H06FGEhIS8LMbMDCEhIThw4IDG12zatAlBQUEYNGgQ3NzcUKtWLUybNg0KhUJtu8uXL8PT0xOVKlVC9+7dkZSUVKzHQkRExSs+HujUST38AEBystgeH6//93R3d1fdnJycIJPJVI8vXLgABwcH/PXXX2jQoAGsrKywd+9eXL16Fe3atYObmxvs7e3RqFEjtSsSQN5LYDKZDL/88gvat28PW1tbVKlSBZs2bVI9//olsGXLlsHZ2Rnbtm1D9erVYW9vj9atW6sFtpycHAwdOhTOzs4oV64cRo8ejcjISISHh+d7vPfv30e3bt3g5eUFW1tbBAQEYM2aNWrb5ObmYvr06ahcuTKsrKxQsWJFTJ06VfX8rVu30K1bN5QtWxZ2dnZo2LAhDh06VIhPv/hJFoDS0tKgUCjg5uam1u7m5oaUlBSNr7l27RrWr18PhUKBLVu2IDo6Gj/88AO++eYb1TaBgYFYtmwZtm7digULFiAxMRHBwcF4/PhxvrVkZ2cjIyND7UZERIZBoRDP/AhC3ueUbcOHi9uVtDFjxuDbb7/F+fPnUbt2bWRmZqJt27ZISEjA8ePH0bp1a4SFhb3xD/HJkyejS5cuOHXqFNq2bYvu3bvjwYMH+W7/5MkTzJgxAytWrMDu3buRlJSEUaNGqZ7/7rvvsGrVKixduhT79u1DRkYGNm7cWGANz549Q4MGDbB582acOXMGAwYMQM+ePdX65o4dOxbffvstoqOjce7cOaxevVr1PZ6ZmYkWLVogOTkZmzZtwsmTJ/Hll18iNzdXi09SAoJEkpOTBQDC/v371dq/+OILoXHjxhpfU6VKFcHb21vIyclRtf3www+Cu7t7vu/z8OFDwdHRUfjll1/y3WbixIkCgDy39PR0HY+KiIg0efr0qXDu3Dnh6dOnOr92505BEKNOwbedO/VetsrSpUsFJyenV2raKQAQNm7c+MbX1qxZU5g7d67qsY+PjzBr1izVYwDC+PHjVY8zMzMFAMJff/2l9l4PHz5U1QJAuHLliuo18+fPF9zc3FSP3dzchO+//171OCcnR6hYsaLQrl07bQ9ZEARB+OCDD4SRI0cKgiAIGRkZgpWVlfDzzz9r3Pann34SHBwchPv37+v0Hroq6HcpPT1d6+9vyc4Aubi4QC6XIzU1Va09NTU132udHh4eeOutt9QmPqpevTpSUlLw/Plzja9xdnbGW2+9hStXruRby9ixY5Genq663bx5sxBHRERExeGVKzt62U6fGjZsqPY4MzMTo0aNQvXq1eHs7Ax7e3ucP3/+jWeAateurbpvZ2cHR0dH1YzHmtja2sLf31/12MPDQ7V9eno6UlNT0bhxY9XzcrkcDRo0KLAGhUKBKVOmICAgAGXLloW9vT22bdumqv38+fPIzs5Gq1atNL7+xIkTqFevHsqWLVvg+xgKyQKQpaUlGjRogISEBFVbbm4uEhISEBQUpPE1TZs2xZUrV9ROp126dAkeHh6wtLTU+JrMzExcvXq1wCmzrays4OjoqHYjIiLDoO2KB0VcGaFQXh/NNWrUKGzYsAHTpk3Dnj17cOLECQQEBOT7R7qScmCPkkwmK/DSkabtBU3XCHXw/fffY/bs2Rg9ejR27tyJEydOIDQ0VFW7chbm/LzpeUMj6TxAUVFR+Pnnn7F8+XKcP38eAwcORFZWlmpUWK9evTB27FjV9gMHDsSDBw8wbNgwXLp0CZs3b8a0adMwaNAg1TajRo3Cv//+i+vXr2P//v1o37495HI5unXrVuLHR0RERRccLI72ym9UukwGeHuL20lt37596N27N9q3b4+AgAC4u7vj+vXrJVqDk5MT3NzccOTIEVWbQqHAsWPHCnzdvn370K5dO/To0QN16tRBpUqVcOnSJdXzVapUgY2NjdqJi1fVrl0bJ06cKLDvkiGRdBh8REQE7t27hwkTJiAlJQV169bF1q1bVR2qkpKSYGb2MqN5e3tj27ZtGDFiBGrXrg0vLy8MGzYMo0ePVm2j7IF+//59uLq6olmzZjh48KDOww6JiMgwyOXiUPdOncSw8+qJDmUoio0tvvmAdFGlShXEx8cjLCwMMpkM0dHRknQCHjJkCGJiYlC5cmVUq1YNc+fOxcOHDwuc26hKlSpYv3499u/fjzJlymDmzJlITU1FjRo1AADW1tYYPXo0vvzyS1haWqJp06a4d+8ezp49i379+qFbt26YNm0awsPDERMTAw8PDxw/fhyenp75XtmRkuQzBA4ePBiDBw/W+NyuXbvytAUFBeHgwYP57m/t2rX6Ko2IiAxEhw7A+vWa5wGKjS2eeYAKY+bMmejbty+aNGkCFxcXjB49WpKRxaNHj0ZKSgp69eoFuVyOAQMGIDQ0tMDFQ8ePH49r164hNDQUtra2GDBgAMLDw5Genq7aJjo6Gubm5pgwYQJu374NDw8PfPbZZwDEri1///03Ro4cibZt2yInJwc1atTA/Pnzi/14C0MmFPWiYSmUkZEBJycnpKensz8QEZEePHv2DImJifDz84O1tXWh91OSM0GXJrm5uahevTq6dOmCKVOmSF1OkRT0u6TL97fkZ4CIiIi0JZcDLVtKXYXhu3HjBv7++2+0aNEC2dnZmDdvHhITE/Hxxx9LXZrBkHwxVCIiItIvMzMzLFu2DI0aNULTpk1x+vRp7NixA9WrV5e6NIPBM0BERESljLe3N/bt2yd1GQaNZ4CIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiKiYtSyZUsMHz5c9djX1xexsbEFvkYmk2Hjxo1Ffm997ac0YgAiIiLSICwsDK1bt9b43J49eyCTyXDq1Cmd93vkyBEMGDCgqOWpmTRpEurWrZun/c6dO2jTpo1e36u0YAAiIiLSoF+/fti+fTtuvbr66v9bunQpGjZsiNq1a+u8X1dXV9ja2uqjxDdyd3eHlZVVibyXsWEAIiIi0uDDDz+Eq6srli1bptaemZmJdevWoV+/frh//z66desGLy8v2NraIiAgAGvWrClwv69fArt8+TKaN28Oa2tr1KhRA9u3b8/zmtGjR+Ott96Cra0tKlWqhOjoaLx48QIAsGzZMkyePBknT56ETCaDTCZT1fz6JbDTp0/j3XffhY2NDcqVK4cBAwYgMzNT9Xzv3r0RHh6OGTNmwMPDA+XKlcOgQYNU76XJ1atX0a5dO7i5ucHe3h6NGjXCjh071LbJzs7G6NGj4e3tDSsrK1SuXBmLFy9WPX/27Fl8+OGHcHR0hIODA4KDg3H16tUCP8ei4lIYREQkCUEAnjwp+fe1tQVksjdvZ25ujl69emHZsmUYN24cZP//onXr1kGhUKBbt27IzMxEgwYNMHr0aDg6OmLz5s3o2bMn/P390bhx4ze+R25uLjp06AA3NzccOnQI6enpav2FlBwcHLBs2TJ4enri9OnT6N+/PxwcHPDll18iIiICZ86cwdatW1XBw8nJKc8+srKyEBoaiqCgIBw5cgR3797FJ598gsGDB6uFvJ07d8LDwwM7d+7ElStXEBERgbp166J///4ajyEzMxNt27bF1KlTYWVlhV9//RVhYWG4ePEiKlasCADo1asXDhw4gDlz5qBOnTpITExEWloaACA5ORnNmzdHy5Yt8c8//8DR0RH79u1DTk7OGz+/IhEoj/T0dAGAkJ6ertf95uQIws6dgrB6tfhvTo5ed09EZLCePn0qnDt3Tnj69KmqLTNTEMQYVLK3zEzt6z5//rwAQNi5c6eqLTg4WOjRo0e+r/nggw+EkSNHqh63aNFCGDZsmOqxj4+PMGvWLEEQBGHbtm2Cubm5kJycrHr+r7/+EgAIGzZsyPc9vv/+e6FBgwaqxxMnThTq1KmTZ7tX97No0SKhTJkyQuYrH8DmzZsFMzMzISUlRRAEQYiMjBR8fHyEnFe+oDp37ixERETkW4smNWvWFObOnSsIgiBcvHhRACBs375d47Zjx44V/Pz8hOfPn2u1b02/S0q6fH/zElgJiY8HfH2Bd94BPv5Y/NfXV2wnIiLDVK1aNTRp0gRLliwBAFy5cgV79uxBv379AAAKhQJTpkxBQEAAypYtC3t7e2zbtg1JSUla7f/8+fPw9vaGp6enqi0oKCjPdnFxcWjatCnc3d1hb2+P8ePHa/0er75XnTp1YGdnp2pr2rQpcnNzcfHiRVVbzZo1IZfLVY89PDxw9+7dfPebmZmJUaNGoXr16nB2doa9vT3Onz+vqu/EiROQy+Vo0aKFxtefOHECwcHBsLCw0Ol4ioqXwEpAfDzQqZP4t8erkpPF9vXrgQ4dpKmNiEgqtrbAK91PSvR9ddGvXz8MGTIE8+fPx9KlS+Hv76/6Mv/+++8xe/ZsxMbGIiAgAHZ2dhg+fDieP3+ut3oPHDiA7t27Y/LkyQgNDYWTkxPWrl2LH374QW/v8arXg4hMJkNubm6+248aNQrbt2/HjBkzULlyZdjY2KBTp06qz8DGxqbA93vT88WFAaiYKRTAsGF5ww8gtslkwPDhQLt2wCuBm4io1JPJgFdORhisLl26YNiwYVi9ejV+/fVXDBw4UNUfaN++fWjXrh169OgBQOzTc+nSJdSoUUOrfVevXh03b97EnTt34OHhAQA4ePCg2jb79++Hj48Pxo0bp2q7ceOG2jaWlpZQKBRvfK9ly5YhKytLdRZo3759MDMzQ9WqVbWqV5N9+/ahd+/eaN++PQDxjND169dVzwcEBCA3Nxf//vsvQkJC8ry+du3aWL58OV68eFGiZ4F4CayY7dkDaBhBqSIIwM2b4nZERGR47O3tERERgbFjx+LOnTvo3bu36rkqVapg+/bt2L9/P86fP49PP/0UqampWu87JCQEb731FiIjI3Hy5Ens2bNHLego3yMpKQlr167F1atXMWfOHGzYsEFtG19fXyQmJuLEiRNIS0tDdnZ2nvfq3r07rK2tERkZiTNnzmDnzp0YMmQIevbsCTc3N90+lNfqi4+Px4kTJ3Dy5El8/PHHameMfH19ERkZib59+2Ljxo1ITEzErl278NtvvwEABg8ejIyMDHTt2hX//fcfLl++jBUrVqhdlisODEDF7M4d/W5HREQlr1+/fnj48CFCQ0PV+uuMHz8e9evXR2hoKFq2bAl3d3eEh4drvV8zMzNs2LABT58+RePGjfHJJ59g6tSpatt89NFHGDFiBAYPHoy6deti//79iI6OVtumY8eOaN26Nd555x24urpqHIpva2uLbdu24cGDB2jUqBE6deqEVq1aYd68ebp9GK+ZOXMmypQpgyZNmiAsLAyhoaGoX7++2jYLFixAp06d8Pnnn6NatWro378/srKyAADlypXDP//8g8zMTLRo0QINGjTAzz//XOxng2SCoOnijGnLyMiAk5MT0tPT4ejoWKR97doldnh+k507gZYti/RWREQG69mzZ0hMTISfnx+sra2lLoeMWEG/S7p8f/MMUDELDgYqVMh/zgmZDPD2FrcjIiKiksEAVMzkcmD2bPH+6yFI+Tg2lh2giYiIShIDUAno0EEc6u7lpd5eoQKHwBMREUmBw+BLSIcO4lD3PXvEDs8eHuJlL575ISIiKnkMQCVILmdHZyIiIkPAS2BERFRiOPCYikpfv0MMQEREVOyUc7o8kWL5dypVlL9DRZ0niJfAiIio2Mnlcjg7O6sW1bS1tVUtJ0GkDUEQ8OTJE9y9exfOzs5qC7YWBgMQERGVCHd3dwAocGVxojdxdnZW/S4VBQMQERGVCJlMBg8PD5QvXx4vXryQuhwyQhYWFkU+86PEAERERCVKLpfr7UuMqLDYCZqIiIhMDgMQERERmRwGICIiIjI57AOkgXKSpYyMDIkrISIiIm0pv7e1mSyRAUiDx48fAwC8vb0lroSIiIh09fjxYzg5ORW4jUzgvOR55Obm4vbt23BwcOBEXfnIyMiAt7c3bt68CUdHR6nLMXn8eRgW/jwMC38ehqU4fx6CIODx48fw9PSEmVnBvXx4BkgDMzMzVKhQQeoyjIKjoyP/h2JA+PMwLPx5GBb+PAxLcf083nTmR4mdoImIiMjkMAARERGRyWEAokKxsrLCxIkTYWVlJXUpBP48DA1/HoaFPw/DYig/D3aCJiIiIpPDM0BERERkchiAiIiIyOQwABEREZHJYQAiIiIik8MARFqLiYlBo0aN4ODggPLlyyM8PBwXL16Uuiz6f99++y1kMhmGDx8udSkmLTk5GT169EC5cuVgY2ODgIAA/Pfff1KXZZIUCgWio6Ph5+cHGxsb+Pv7Y8qUKVqtE0VFt3v3boSFhcHT0xMymQwbN25Ue14QBEyYMAEeHh6wsbFBSEgILl++XGL1MQCR1v79918MGjQIBw8exPbt2/HixQu8//77yMrKkro0k3fkyBH89NNPqF27ttSlmLSHDx+iadOmsLCwwF9//YVz587hhx9+QJkyZaQuzSR99913WLBgAebNm4fz58/ju+++w/Tp0zF37lypSzMJWVlZqFOnDubPn6/x+enTp2POnDlYuHAhDh06BDs7O4SGhuLZs2clUh+HwVOh3bt3D+XLl8e///6L5s2bS12OycrMzET9+vXx448/4ptvvkHdunURGxsrdVkmacyYMdi3bx/27NkjdSkE4MMPP4SbmxsWL16sauvYsSNsbGywcuVKCSszPTKZDBs2bEB4eDgA8eyPp6cnRo4ciVGjRgEA0tPT4ebmhmXLlqFr167FXhPPAFGhpaenAwDKli0rcSWmbdCgQfjggw8QEhIidSkmb9OmTWjYsCE6d+6M8uXLo169evj555+lLstkNWnSBAkJCbh06RIA4OTJk9i7dy/atGkjcWWUmJiIlJQUtf9vOTk5ITAwEAcOHCiRGrgYKhVKbm4uhg8fjqZNm6JWrVpSl2Oy1q5di2PHjuHIkSNSl0IArl27hgULFiAqKgpfffUVjhw5gqFDh8LS0hKRkZFSl2dyxowZg4yMDFSrVg1yuRwKhQJTp05F9+7dpS7N5KWkpAAA3Nzc1Nrd3NxUzxU3BiAqlEGDBuHMmTPYu3ev1KWYrJs3b2LYsGHYvn07rK2tpS6HIP5h0LBhQ0ybNg0AUK9ePZw5cwYLFy5kAJLAb7/9hlWrVmH16tWoWbMmTpw4geHDh8PT05M/D+IlMNLd4MGD8eeff2Lnzp2oUKGC1OWYrKNHj+Lu3buoX78+zM3NYW5ujn///Rdz5syBubk5FAqF1CWaHA8PD9SoUUOtrXr16khKSpKoItP2xRdfYMyYMejatSsCAgLQs2dPjBgxAjExMVKXZvLc3d0BAKmpqWrtqampqueKGwMQaU0QBAwePBgbNmzAP//8Az8/P6lLMmmtWrXC6dOnceLECdWtYcOG6N69O06cOAG5XC51iSanadOmeaaGuHTpEnx8fCSqyLQ9efIEZmbqX3NyuRy5ubkSVURKfn5+cHd3R0JCgqotIyMDhw4dQlBQUInUwEtgpLVBgwZh9erV+OOPP+Dg4KC6Tuvk5AQbGxuJqzM9Dg4Oefpf2dnZoVy5cuyXJZERI0agSZMmmDZtGrp06YLDhw9j0aJFWLRokdSlmaSwsDBMnToVFStWRM2aNXH8+HHMnDkTffv2lbo0k5CZmYkrV66oHicmJuLEiRMoW7YsKlasiOHDh+Obb75BlSpV4Ofnh+joaHh6eqpGihU7gUhLADTeli5dKnVp9P9atGghDBs2TOoyTNr//vc/oVatWoKVlZVQrVo1YdGiRVKXZLIyMjKEYcOGCRUrVhSsra2FSpUqCePGjROys7OlLs0k7Ny5U+N3RmRkpCAIgpCbmytER0cLbm5ugpWVldCqVSvh4sWLJVYf5wEiIiIik8M+QERERGRyGICIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIqJ8yGQybNy4UeoyiKgYMAARkUHq3bs3ZDJZnlvr1q2lLo2ISgGuBUZEBqt169ZYunSpWpuVlZVE1RBRacIzQERksKysrODu7q52K1OmDADx8tSCBQvQpk0b2NjYoFKlSli/fr3a60+fPo13330XNjY2KFeuHAYMGIDMzEy1bZYsWYKaNWvCysoKHh4eGDx4sNrzaWlpaN++PWxtbVGlShVs2rRJ9dzDhw/RvXt3uLq6wsbGBlWqVMkT2IjIMDEAEZHRio6ORseOHXHy5El0794dXbt2xfnz5wEAWVlZCA0NRZkyZXDkyBGsW7cOO3bsUAs4CxYswKBBgzBgwACcPn0amzZtQuXKldXeY/LkyejSpQtOnTqFtm3bonv37njw4IHq/c+dO4e//voL58+fx4IFC+Di4lJyHwARFV6JLbtKRKSDyMhIQS6XC3Z2dmq3qVOnCoIgCACEzz77TO01gYGBwsCBAwVBEIRFixYJZcqUETIzM1XPb968WTAzMxNSUlIEQRAET09PYdy4cfnWAEAYP3686nFmZqYAQPjrr78EQRCEsLAwoU+fPvo5YCIqUewDREQG65133sGCBQvU2sqWLau6HxQUpPZcUFAQTpw4AQA4f/486tSpAzs7O9XzTZs2RW5uLi5evAiZTIbbt2+jVatWBdZQu3Zt1X07Ozs4Ojri7t27AICBAweiY8eOOHbsGN5//32Eh4ejSZMmhTpWIipZDEBEZLDs7OzyXJLSFxsbG622s7CwUHssk8mQm5sLAGjTpg1u3LiBLVu2YPv27WjVqhUGDRqEGTNm6L1eItIv9gEiIqN18ODBPI+rV68OAKhevTpOnjyJrKws1fP79u2DmZkZqlatCgcHB/j6+iIhIaFINbi6uiIyMhIrV65EbGwsFi1aVKT9EVHJ4BkgIjJY2dnZSElJUWszNzdXdTRet24dGjZsiGbNmmHVqlU4fPgwFi9eDADo3r07Jk6ciMjISEyaNAn37t3DkCFD0LNnT7i5uQEAJk2ahM8++wzly5dHmzZt8PjxY+zbtw9DhgzRqr4JEyagQYMGqFmzJrKzs/Hnn3+qAhgRGTYGICIyWFu3boWHh4daW9WqVXHhwgUA4gittWvX4vPPP4eHhwfWrFmDGjVqAABsbW2xbds2DBs2DI0aNYKtrS06duyImTNnqvYVGRmJZ8+eYdasWRg1ahRcXFzQqVMnreuztLTE2LFjcf36ddjY2CA4OBhr167Vw5ETUXGTCYIgSF0EEZGuZDIZNmzYgPDwcKlLISIjxD5AREREZHIYgIiIiMjksA8QERklXr0noqLgGSAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOf8Hw4fJoTNf+rYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7865d6f2",
   "metadata": {},
   "source": [
    "### Export the model\n",
    "\n",
    "We can export the model including the TextVectorization layer inside the model to conduct inference on raw text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "93b0a42c-437e-41bb-99e7-d58cb8036a3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - accuracy: 0.4935 - binary_accuracy: 0.0000e+00 - loss: 0.0000e+00\n",
      "{'accuracy': 0.5, 'binary_accuracy': 0.0, 'loss': 0.0}\n"
     ]
    }
   ],
   "source": [
    "export_model = tf.keras.Sequential([\n",
    "  vectorize_layer,\n",
    "  model,\n",
    "  layers.Activation('sigmoid')\n",
    "])\n",
    "\n",
    "export_model.compile(\n",
    "    loss=losses.BinaryCrossentropy(from_logits=False), optimizer=\"adam\", metrics=['accuracy']\n",
    ")\n",
    "\n",
    "# Test it with `raw_test_ds`, which yields raw strings\n",
    "metrics = export_model.evaluate(raw_test_ds, return_dict=True)\n",
    "print(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0795584",
   "metadata": {},
   "source": [
    "Conduct inference on new data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8939539b-a600-48b1-a55e-3f1087f4a855",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 61ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.67346764],\n",
       "       [0.634105  ],\n",
       "       [0.61044645]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "examples = tf.constant([\n",
    "  \"The movie was great!\",\n",
    "  \"The movie was okay.\",\n",
    "  \"The movie was terrible...\"\n",
    "])\n",
    "\n",
    "export_model.predict(examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6b40a59-8d3b-44ec-a4f7-92c5742a0c1c",
   "metadata": {},
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3e520822",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir('models') if not os.path.exists('models') else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7f22cc32-2708-4808-8e76-99024da87a21",
   "metadata": {},
   "outputs": [],
   "source": [
    "export_model.save('models/text_model.keras')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0461f74-fdd0-4f30-9f44-0be7ad00d9b0",
   "metadata": {},
   "source": [
    "### Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c9cf2c7f-5e86-4ff8-984e-dd0ed7a3ece9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"sequential_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ text_vectorization              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">250</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ sequential (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Sequential</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │       <span style=\"color: #00af00; text-decoration-color: #00af00\">160,017</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ activation (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ text_vectorization              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m250\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
       "│ (\u001b[38;5;33mTextVectorization\u001b[0m)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ sequential (\u001b[38;5;33mSequential\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │       \u001b[38;5;34m160,017\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ activation (\u001b[38;5;33mActivation\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │             \u001b[38;5;34m0\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">160,017</span> (625.07 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m160,017\u001b[0m (625.07 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">160,017</span> (625.07 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m160,017\u001b[0m (625.07 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# register callables as custom objects before loading\n",
    "custom_objects = {\"vectorize_layer\": vectorize_layer, \"custom_standardization\": custom_standardization}\n",
    "with tf.keras.utils.custom_object_scope(custom_objects):\n",
    "    new_model = tf.keras.models.load_model('models/text_model.keras', compile=False)\n",
    "\n",
    "new_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "242a4f7e-fa45-4d21-b103-fe3718bc0f10",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "531680b2-42ef-4205-9a38-6995aee9f340",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 59ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.67346764],\n",
       "       [0.634105  ],\n",
       "       [0.61044645]], dtype=float32)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model.predict(examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a82ae387-1587-4175-b4b2-66586e4668f7",
   "metadata": {},
   "source": [
    "## PySpark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d6d515c2-ce53-4af5-a936-ae91fdecea99",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.functions import predict_batch_udf\n",
    "from pyspark.sql.functions import struct, col, array, pandas_udf\n",
    "from pyspark.sql.types import ArrayType, FloatType, DoubleType\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkConf\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39c35256",
   "metadata": {},
   "source": [
    "Check the cluster environment to handle any platform-specific Spark configurations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "31de0c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n",
    "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n",
    "on_standalone = not (on_databricks or on_dataproc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55ad7f00",
   "metadata": {},
   "source": [
    "#### Create Spark Session\n",
    "\n",
    "For local standalone clusters, we'll connect to the cluster and create the Spark Session.  \n",
    "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6b653c43",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25/02/04 14:05:31 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n",
      "25/02/04 14:05:31 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "25/02/04 14:05:31 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "conf = SparkConf()\n",
    "\n",
    "if 'spark' not in globals():\n",
    "    if on_standalone:\n",
    "        import socket\n",
    "        \n",
    "        conda_env = os.environ.get(\"CONDA_PREFIX\")\n",
    "        hostname = socket.gethostname()\n",
    "        conf.setMaster(f\"spark://{hostname}:7077\")\n",
    "        conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n",
    "        conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n",
    "    elif on_dataproc:\n",
    "        conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n",
    "\n",
    "    conf.set(\"spark.executor.cores\", \"8\")\n",
    "    conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n",
    "    conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n",
    "    conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n",
    "    conf.set(\"spark.python.worker.reuse\", \"true\")\n",
    "\n",
    "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n",
    "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b39d27",
   "metadata": {},
   "source": [
    "Load the IMDB dataset. We'll perform inference on the first sentence of each sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ef3309eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"imdb\", split=\"test\")\n",
    "dataset = dataset.to_pandas().drop(columns=\"label\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a7672d1",
   "metadata": {},
   "source": [
    "#### Create PySpark DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bb05466f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StructType([StructField('text', StringType(), True)])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = spark.createDataFrame(dataset).repartition(8)\n",
    "df.schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3f0a594b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25/02/04 14:05:36 WARN TaskSetManager: Stage 0 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.<br /><br />The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.<br /><br />The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.<br /><br />I really got nothing much left to say except, give us back CKY2K, cause Bam suck..<br /><br />I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9d9db063",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25/02/04 14:05:37 WARN TaskSetManager: Stage 3 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    }
   ],
   "source": [
    "data_path = \"spark-dl-datasets/imdb_test\"\n",
    "if on_databricks:\n",
    "    data_path = \"dbfs:/FileStore/\" + data_path\n",
    "\n",
    "df.write.mode(\"overwrite\").parquet(data_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f78a16a",
   "metadata": {},
   "source": [
    "#### Load and Preprocess PySpark DataFrame\n",
    "\n",
    "Define our preprocess function. We'll take the first sentence of each sample as our input for sentiment analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1c081557",
   "metadata": {},
   "outputs": [],
   "source": [
    "@pandas_udf(\"string\")\n",
    "def preprocess(text: pd.Series) -> pd.Series:\n",
    "    return pd.Series([s.split(\".\")[0] for s in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "60af570a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Limit to N rows, since this can be slow\n",
    "df = spark.read.parquet(data_path).limit(512).repartition(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a690f6df",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_df = df.select(preprocess(col(\"text\")).alias(\"lines\")).cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01166d97",
   "metadata": {},
   "source": [
    "## Inference using Spark DL API\n",
    "\n",
    "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n",
    "\n",
    "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n",
    "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7b7a8395-e2ae-4c3c-bf57-763dfde600ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_model_path = \"{}/models/text_model.keras\".format(os.getcwd())\n",
    "\n",
    "# For cloud environments, copy the model to the distributed file system.\n",
    "if on_databricks:\n",
    "    dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n",
    "    dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/text_model.keras\"\n",
    "    shutil.copy(text_model_path, dbfs_model_path)\n",
    "    text_model_path = dbfs_model_path\n",
    "elif on_dataproc:\n",
    "    # GCS is mounted at /mnt/gcs by the init script\n",
    "    models_dir = \"/mnt/gcs/spark-dl/models\"\n",
    "    os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n",
    "    gcs_model_path = models_dir + \"/text_model.keras\"\n",
    "    shutil.copy(text_model_path, gcs_model_path)\n",
    "    text_model_path = gcs_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8c0524cf-3a75-4fb8-8025-f0654acce13e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_batch_fn():\n",
    "    # since this function runs on the executor, any required imports should be added inside the function.\n",
    "    import re\n",
    "    import string\n",
    "    import tensorflow as tf\n",
    "    from tensorflow.keras import layers\n",
    "\n",
    "    # Enable GPU memory growth to avoid CUDA OOM\n",
    "    gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "    if gpus:\n",
    "        try:\n",
    "            for gpu in gpus:\n",
    "                tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        except RuntimeError as e:\n",
    "            print(e)\n",
    "\n",
    "    def custom_standardization(input_data):\n",
    "        lowercase = tf.strings.lower(input_data)\n",
    "        stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n",
    "        return tf.strings.regex_replace(\n",
    "            stripped_html, \"[%s]\" % re.escape(string.punctuation), \"\"\n",
    "        )\n",
    "\n",
    "    max_features = 10000\n",
    "    sequence_length = 250\n",
    "\n",
    "    vectorize_layer = layers.TextVectorization(\n",
    "        standardize=custom_standardization,\n",
    "        max_tokens=max_features,\n",
    "        output_mode=\"int\",\n",
    "        output_sequence_length=sequence_length,\n",
    "    )\n",
    "\n",
    "    custom_objects = {\"vectorize_layer\": vectorize_layer,\n",
    "                      \"custom_standardization\": custom_standardization}\n",
    "    with tf.keras.utils.custom_object_scope(custom_objects):\n",
    "        model = tf.keras.models.load_model(text_model_path)\n",
    "\n",
    "    def predict(inputs):\n",
    "        return model.predict(inputs)\n",
    "\n",
    "    return predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0d603644-d938-4c87-aa8a-2512251638d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "classify = predict_batch_udf(predict_batch_fn,\n",
    "                             return_type=FloatType(),\n",
    "                             batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "0b480622-8dc1-4879-933e-c43112768630",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 9:>                                                          (0 + 8) / 8]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.81 ms, sys: 3.75 ms, total: 10.6 ms\n",
      "Wall time: 4.62 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(struct(\"lines\")))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "31b0a262-387e-4a5e-a60e-b9b8ee456199",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.58 ms, sys: 0 ns, total: 4.58 ms\n",
      "Wall time: 142 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(\"lines\"))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "7ef9e431-59f5-4b29-9f79-ae16a9cfb0b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 903 μs, sys: 4.09 ms, total: 5 ms\n",
      "Wall time: 222 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(col(\"lines\")))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9a325ee2-3268-414a-bb75-a5fcf794f512",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------------------------------------------------------------+----------+\n",
      "|                                                                           lines|     preds|\n",
      "+--------------------------------------------------------------------------------+----------+\n",
      "|The only reason I'm even giving this movie a 4 is because it was made in to a...|  0.571606|\n",
      "|Awkward disaster mishmash has a team of scavengers coming across the overturn...| 0.6264358|\n",
      "|Here is a fantastic concept for a film - a series of meteors crash into a sma...| 0.6764294|\n",
      "|              I walked out of the cinema having suffered this film after 30 mins| 0.6258814|\n",
      "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...|0.63658905|\n",
      "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...|  0.633625|\n",
      "|                                                                     A good cast|0.65998995|\n",
      "|Yet again, I appear to be the only person on planet Earth who is capable of c...| 0.6435825|\n",
      "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.6453945|\n",
      "|Upon writing this review I have difficulty trying to think of what to write a...|0.61587423|\n",
      "|                                                                    Simply awful|  0.594154|\n",
      "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.6366444|\n",
      "|                                                                            Well|0.65976477|\n",
      "|                                                This is a new approach to comedy| 0.6555772|\n",
      "|     It's been mentioned by others the inane dialogue in this series and I agree| 0.6534178|\n",
      "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.5919746|\n",
      "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.6527056|\n",
      "|                                       1983's \"Frightmare\" is an odd little film|0.64622015|\n",
      "|                                                           'Felony' is a B-movie|0.64882356|\n",
      "|                                          This movie defines the word \"confused\"|0.63689107|\n",
      "+--------------------------------------------------------------------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show(truncate=80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad9b07e6",
   "metadata": {},
   "source": [
    "## Using Triton Inference Server\n",
    "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL.  \n",
    "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server.  \n",
    "\n",
    "The process looks like this:\n",
    "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n",
    "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n",
    "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n",
    "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n",
    "\n",
    "<img src=\"../images/spark-server.png\" alt=\"drawing\" width=\"700\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "889a1623",
   "metadata": {},
   "source": [
    "First we'll cleanup the vocabulary layer of the model to remove non-ASCII characters. This ensures the inputs can be properly serialized and sent to Triton."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f4f14c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "\n",
    "def normalize_vocabulary(vocab):\n",
    "    # Normalize each word in the vocabulary to remove non-ASCII characters\n",
    "    normalized_vocab = [\n",
    "        unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8')\n",
    "        for word in vocab\n",
    "    ]\n",
    "    normalized_vocab = filter(lambda x: x != '', normalized_vocab)\n",
    "    normalized_vocab = list(set(normalized_vocab)) \n",
    "\n",
    "\n",
    "    return normalized_vocab\n",
    "\n",
    "vocab = vectorize_layer.get_vocabulary()\n",
    "normalized_vocab = normalize_vocabulary(vocab)\n",
    "\n",
    "# Reassign the cleaned vocabulary to the TextVectorization layer\n",
    "vectorize_layer.set_vocabulary(normalized_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "9614a192",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model with the cleaned vocabulary\n",
    "triton_model_path = '{}/models/text_model_cleaned.keras'.format(os.getcwd())\n",
    "export_model.save(triton_model_path)\n",
    "\n",
    "# For cloud environments, copy the model to the distributed file system.\n",
    "if on_databricks:\n",
    "    dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n",
    "    dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/text_model_cleaned.keras\"\n",
    "    shutil.copy(triton_model_path, dbfs_model_path)\n",
    "    triton_model_path = dbfs_model_path\n",
    "elif on_dataproc:\n",
    "    # GCS is mounted at /mnt/gcs by the init script\n",
    "    models_dir = \"/mnt/gcs/spark-dl/models\"\n",
    "    os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n",
    "    gcs_model_path = models_dir + \"/text_model_cleaned.keras\"\n",
    "    shutil.copy(triton_model_path, gcs_model_path)\n",
    "    triton_model_path = gcs_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "32d0142a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edddffb9",
   "metadata": {},
   "source": [
    "Import the helper class from server_utils.py:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "444bad3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.addPyFile(\"server_utils.py\")\n",
    "\n",
    "from server_utils import TritonServerManager"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0923a56",
   "metadata": {},
   "source": [
    "Define the Triton Server function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "a4d37d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "def triton_server(ports, model_path):\n",
    "    import time\n",
    "    import signal\n",
    "    import numpy as np\n",
    "    import tensorflow as tf\n",
    "    from pytriton.decorators import batch\n",
    "    from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n",
    "    from pytriton.triton import Triton, TritonConfig\n",
    "    from pyspark import TaskContext\n",
    "    from tensorflow.keras import layers \n",
    "\n",
    "    \n",
    "    print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n",
    "    # Enable GPU memory growth\n",
    "    gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "    if gpus:\n",
    "        try:\n",
    "            for gpu in gpus:\n",
    "                tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        except RuntimeError as e:\n",
    "            print(e)\n",
    "\n",
    "    def custom_standardization(input_data):\n",
    "        lowercase = tf.strings.lower(input_data)\n",
    "        stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n",
    "        return tf.strings.regex_replace(\n",
    "            stripped_html, \"[%s]\" % re.escape(string.punctuation), \"\"\n",
    "        )\n",
    "\n",
    "    max_features = 10000\n",
    "    sequence_length = 250\n",
    "\n",
    "    vectorize_layer = layers.TextVectorization(\n",
    "        standardize=custom_standardization,\n",
    "        max_tokens=max_features,\n",
    "        output_mode=\"int\",\n",
    "        output_sequence_length=sequence_length,\n",
    "    )\n",
    "\n",
    "    custom_objects = {\"vectorize_layer\": vectorize_layer,\n",
    "                \"custom_standardization\": custom_standardization}\n",
    "\n",
    "    with tf.keras.utils.custom_object_scope(custom_objects):\n",
    "        model = tf.keras.models.load_model(model_path)\n",
    "\n",
    "    @batch\n",
    "    def _infer_fn(**inputs):\n",
    "        sentences = inputs[\"text\"]\n",
    "        print(f\"SERVER: Received batch of size {len(sentences)}.\")\n",
    "        decoded_sentences = tf.convert_to_tensor(np.vectorize(lambda x: x.decode('utf-8'))(sentences))\n",
    "        return {\n",
    "            \"preds\": model.predict(decoded_sentences)\n",
    "        }\n",
    "    \n",
    "    workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n",
    "    triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n",
    "    with Triton(config=triton_conf, workspace=workspace_path) as triton:\n",
    "        triton.bind(\n",
    "            model_name=\"TextModel\",\n",
    "            infer_func=_infer_fn,\n",
    "            inputs=[\n",
    "                Tensor(name=\"text\", dtype=np.bytes_, shape=(-1,)),\n",
    "            ],\n",
    "            outputs=[\n",
    "                Tensor(name=\"preds\", dtype=np.float32, shape=(-1,)),\n",
    "            ],\n",
    "            config=ModelConfig(\n",
    "                max_batch_size=128,\n",
    "                batcher=DynamicBatcher(max_queue_delay_microseconds=5000),  # 5ms\n",
    "            ),\n",
    "            strict=True,\n",
    "        )\n",
    "\n",
    "        def _stop_triton(signum, frame):\n",
    "            # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n",
    "            print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n",
    "            triton.stop()\n",
    "\n",
    "        signal.signal(signal.SIGTERM, _stop_triton)\n",
    "\n",
    "        print(\"SERVER: Serving inference\")\n",
    "        triton.serve()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d340e231",
   "metadata": {},
   "source": [
    "#### Start Triton servers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcdb7c5a",
   "metadata": {},
   "source": [
    "The `TritonServerManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n",
    "- Find available ports for HTTP/gRPC/metrics\n",
    "- Deploy a server on each node via stage-level scheduling\n",
    "- Gracefully shutdown servers across nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4d5dc419",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"TextModel\"\n",
    "server_manager = TritonServerManager(model_name=model_name, model_path=triton_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20198644",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n",
      "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n",
    "server_manager.start_servers(triton_server)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1477f4b",
   "metadata": {},
   "source": [
    "#### Define client function"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "798c2815",
   "metadata": {},
   "source": [
    "Get the hostname -> url mapping from the server manager:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813d42cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "host_to_http_url = server_manager.host_to_http_url  # or server_manager.host_to_grpc_url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f16617e3",
   "metadata": {},
   "source": [
    "Define the Triton inference function, which returns a predict function for batch inference through the server:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0ad47438",
   "metadata": {},
   "outputs": [],
   "source": [
    "def triton_fn(model_name, host_to_url):\n",
    "    import socket\n",
    "    import numpy as np\n",
    "    from pytriton.client import ModelClient\n",
    "\n",
    "    url = host_to_url[socket.gethostname()]\n",
    "    print(f\"CLIENT: Connecting to {model_name} at {url}\")\n",
    "\n",
    "    def infer_batch(inputs):\n",
    "        with ModelClient(url, model_name, inference_timeout_s=240) as client:\n",
    "            encoded_inputs = np.vectorize(lambda x: x.encode(\"utf-8\"))(inputs).astype(np.bytes_)\n",
    "            encoded_inputs = np.expand_dims(encoded_inputs, axis=1)\n",
    "            result_data = client.infer_batch(encoded_inputs)\n",
    "            \n",
    "            return result_data[\"preds\"]\n",
    "            \n",
    "    return infer_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "8e06d33f-5cef-4a48-afc3-5d468f8ec2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n",
    "                             return_type=FloatType(),\n",
    "                             batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91974885",
   "metadata": {},
   "source": [
    "#### Load and preprocess DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "41106a02-236e-4cb3-ac51-76aa64b663c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.parquet(data_path).limit(512).repartition(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "e851870b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25/02/04 14:05:48 WARN CacheManager: Asked to cache already cached data.\n"
     ]
    }
   ],
   "source": [
    "input_df = df.select(preprocess(col(\"text\")).alias(\"lines\")).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d89e74ad-e551-4bfa-ad08-98725878630a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 24:==============>                                           (2 + 6) / 8]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.92 ms, sys: 4.06 ms, total: 6.97 ms\n",
      "Wall time: 1.03 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(struct(\"lines\")))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b4fa7fc9-341c-49a6-9af2-e316f2355d67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.39 ms, sys: 2.15 ms, total: 3.53 ms\n",
      "Wall time: 237 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(\"lines\"))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "564f999b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 862 μs, sys: 2.77 ms, total: 3.63 ms\n",
      "Wall time: 225 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = input_df.withColumn(\"preds\", classify(col(\"lines\")))\n",
    "results = predictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "9222e8a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------------------------------------------------------------+----------+\n",
      "|                                                                           lines|     preds|\n",
      "+--------------------------------------------------------------------------------+----------+\n",
      "|The only reason I'm even giving this movie a 4 is because it was made in to a...|0.67212176|\n",
      "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.63807774|\n",
      "|Here is a fantastic concept for a film - a series of meteors crash into a sma...|0.65471745|\n",
      "|              I walked out of the cinema having suffered this film after 30 mins| 0.6527998|\n",
      "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.6405446|\n",
      "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...|0.63534474|\n",
      "|                                                                     A good cast|0.64761806|\n",
      "|Yet again, I appear to be the only person on planet Earth who is capable of c...|0.66956663|\n",
      "|As a serious horror fan, I get that certain marketing ploys are used to sell ...|0.62346375|\n",
      "|Upon writing this review I have difficulty trying to think of what to write a...|  0.681598|\n",
      "|                                                                    Simply awful| 0.6537583|\n",
      "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.6382922|\n",
      "|                                                                            Well|0.65424603|\n",
      "|                                                This is a new approach to comedy| 0.6628315|\n",
      "|     It's been mentioned by others the inane dialogue in this series and I agree|0.63345987|\n",
      "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.6459369|\n",
      "|This movie was playing on Lifetime Movie Network last month and I decided to ...|0.65335083|\n",
      "|                                       1983's \"Frightmare\" is an odd little film|0.65602964|\n",
      "|                                                           'Felony' is a B-movie| 0.6583404|\n",
      "|                                          This movie defines the word \"confused\"| 0.6217103|\n",
      "+--------------------------------------------------------------------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show(truncate=80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d45e8981-ca44-429b-9b37-e04035c3a86b",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Stop Triton Server on each executor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "a71ac9b6-47a2-4306-bc40-9ce7b4e968ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-02-04 14:05:50,166 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n",
      "2025-02-04 14:06:00,351 - INFO - Sucessfully stopped 1 servers.                 \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[True]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "server_manager.stop_servers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "54a90574-7cbb-487b-b7a8-dcda0e6e301f",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n",
    "    spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88e3bfea-a825-46eb-b8c2-921a932c0089",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spark-dl-tf",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
